changelog shortlog graph tags branches changeset files file revisions raw help

Mercurial > core / annotate lisp/lib/dat/csv.lisp

changeset 277: 10faf95f90dd
parent: 78ef6145e272
child: c0fc6b87557f
author: Richard Westhaver <ellis@rwest.io>
date: Sun, 14 Apr 2024 01:19:10 -0400
permissions: -rw-r--r--
description: stream and basic type upgrades. fixed some bugs and improved csv parsing
123
a4ed30cbe083 data testing, added ical and vcard formats
ellis <ellis@rwest.io>
parents: 122
diff changeset
1
 ;;; lib/dat/csv.lisp --- CSV Data Format
a4ed30cbe083 data testing, added ical and vcard formats
ellis <ellis@rwest.io>
parents: 122
diff changeset
2
 
275
78ef6145e272 return of the uri
Richard Westhaver <ellis@rwest.io>
parents: 123
diff changeset
3
 ;; Character Separated Values
123
a4ed30cbe083 data testing, added ical and vcard formats
ellis <ellis@rwest.io>
parents: 122
diff changeset
4
 
a4ed30cbe083 data testing, added ical and vcard formats
ellis <ellis@rwest.io>
parents: 122
diff changeset
5
 ;;; Code:
122
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
6
 (in-package :dat/csv)
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
7
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
8
 (defun parse-number-no-error (string &optional default)
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
9
   (let ((result
277
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
10
           (ignore-errors
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
11
            (parse-number string))))
122
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
12
     (if result
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
13
         result
277
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
14
         default)))
122
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
15
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
16
 (defparameter *csv-separator* #\,)
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
17
 (defparameter *csv-quote* #\")
277
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
18
 (defparameter *csv-print-quote-p* t "print \" when the element is a string?")
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
19
 (defparameter *csv-default-external-format* :utf-8)
122
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
20
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
21
 (defun write-csv-line (record &key stream (delimiter *csv-separator*))
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
22
   "Accept a record and print it in one line as a csv record.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
23
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
24
 A record is a sequence of element. A element can be of any type.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
25
 If record is nil, nothing will be printed.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
26
 If stream is nil (default case), it will return a string, otherwise it will return nil.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
27
 For efficiency reason, no intermediate string will be constructed. "
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
28
   (let ((result
277
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
29
           (with-output-to-string (s)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
30
             (let ((*standard-output* s)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
31
                   (record-size (length record)))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
32
               (loop for e across record
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
33
                     for i from 0
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
34
                     do (typecase e
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
35
                          (string (progn
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
36
                                    (if *csv-print-quote-p*
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
37
                                        (progn
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
38
                                          (write-char *csv-quote*)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
39
                                          (write-string e)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
40
                                          (write-char *csv-quote*))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
41
                                        (write-string e))))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
42
                          (t (princ e)))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
43
                        (when (< i (1- record-size))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
44
                          (write-char delimiter)))))))
122
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
45
     (format stream "~&~a" result)))
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
46
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
47
 (defun write-csv-stream (stream table &key (delimiter *csv-separator*))
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
48
   "Accept a stream and a table and output the table as csv form to the stream.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
49
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
50
 A table is a sequence of lines. A line is a sequence of elements.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
51
 Elements can be any types"
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
52
   (loop for l across table
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
53
         do (write-csv-line l :stream stream :delimiter delimiter))
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
54
   (write-char #\newline stream)
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
55
   '(ok))
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
56
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
57
 (defun write-csv-file (filename table &key (external-format *csv-default-external-format*) (delimiter *csv-separator*))
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
58
   "Accept a filename and a table and output the table as csv form to the file.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
59
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
60
 A table is a sequence of lines. A line is a sequence of elements.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
61
 Elements can be any types"
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
62
   (with-open-file (f filename :direction :output
277
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
63
                               :if-does-not-exist :create
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
64
                               :if-exists :supersede
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
65
                               :external-format external-format)
122
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
66
     (write-csv-stream f table :delimiter delimiter)))
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
67
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
68
 (defun parse-csv-string (str &key (delimiter *csv-separator*)) ;; refer RFC4180
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
69
   (coerce
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
70
    ;; (regexp:split-re "," str)
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
71
    (let ((q-count (count *csv-quote* str :test #'char-equal)))
277
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
72
      (when (oddp q-count) (warn 'simple-warning :format-control "odd number of #\" in a line (~A)"
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
73
                                                 :format-arguments (list q-count)))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
74
      (if (zerop q-count)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
75
          (cl-ppcre:split delimiter str) ;(cl-ppcre:split *csv-separator* str)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
76
          (macrolet ((push-f (fld flds) `(push (coerce (reverse ,fld) 'string) ,flds)))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
77
            (loop with state = :at-first ;; :at-first | :data-nq | :data-q | :q-in-nq | q-in-q
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
78
                  with field with fields
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
79
                  for chr of-type character across str
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
80
                  do (cond ((eq state :at-first)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
81
                            (setf field nil)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
82
                            (cond ((char-equal chr *csv-quote*) (setf state :data-q))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
83
                                  ((char-equal chr delimiter) (push "" fields))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
84
                                  (t (setf state :data-nq) (push chr field))))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
85
                           ((eq state :data-nq)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
86
                            (cond ((char-equal chr *csv-quote*) (setf state :q-in-nq))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
87
                                  ((char-equal chr delimiter)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
88
                                   (push-f field fields)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
89
                                   (setf state :at-first))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
90
                                  (t (push chr field))))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
91
                           ((eq state :q-in-nq)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
92
                            (cond ((char-equal chr *csv-quote*) (setf state :q-in-q))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
93
                                  ((char-equal chr delimiter)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
94
                                   (push-f field fields)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
95
                                   (setf state :at-first))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
96
                                  (t (setf state :data-nq) (push chr field))))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
97
                           ((eq state :data-q)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
98
                            (cond ((char-equal chr *csv-quote*) (setf state :q-in-q))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
99
                                  ((char-equal chr delimiter) (push-f field fields) (setf state :at-first))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
100
                                  (t (push chr field))))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
101
                           ((eq state :q-in-q)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
102
                            (cond ((char-equal chr *csv-quote*) (setf state :data-q))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
103
                                  ;; this should only be done conditionally - early escapes quotes
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
104
                                  ((char-equal chr delimiter)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
105
                                   (push-f field fields)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
106
                                   (setf state :at-first))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
107
                                  (t
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
108
                                   ;; (error "illegal value ( ~A ) after quotation" chr)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
109
                                   (push chr field)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
110
                                   ))))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
111
                  finally (return
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
112
                            (progn (push-f field fields) (reverse fields)))))))
122
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
113
    'vector))
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
114
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
115
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
116
 (defun read-csv-line (stream &key type-conv-fns map-fns (delimiter *csv-separator*) (start 0) end)
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
117
   "Read one line from stream and return a csv record.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
118
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
119
 A CSV record is a vector of elements.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
120
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
121
 type-conv-fns should be a list of functions.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
122
 If type-conv-fns is nil (the default case), then all will be treated
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
123
 as string.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
124
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
125
 map-fns is a list of functions of one argument and output one result.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
126
 each function in it will be applied to the parsed element.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
127
 If map-fns is nil, then nothing will be applied.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
128
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
129
 start and end specifies how many elements per record will be included.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
130
 If start or end is negative, it counts from the end. -1 is the last element.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
131
 "
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
132
   (declare (type (or (simple-array function *) null) type-conv-fns map-fns))
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
133
   (let* ((rline (read-line stream nil nil)))
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
134
     (when rline
277
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
135
       (let* ((line (string-trim '(#\Space #\Newline #\Return) rline))
122
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
136
              (strs (parse-csv-string line :delimiter delimiter))
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
137
              (strs-size (length strs)))
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
138
         (when (< start 0)
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
139
           (setf start (+ start strs-size)))
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
140
         (when (and end (< end 0))
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
141
           (setf end (+ end strs-size)))
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
142
         (setf strs (subseq strs start end))
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
143
         (when type-conv-fns
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
144
           (unless (= (length strs) (length type-conv-fns))
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
145
             (error "Number of type specifier (~a) does not match the number of elements (~a)."
277
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
146
                    (length type-conv-fns) (length strs))))
122
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
147
         (when map-fns
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
148
           (unless (= (length strs) (length map-fns))
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
149
             (error "Number of mapping functions (~a) does not match the number of elements (~a)."
277
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
150
                    (length map-fns) (length strs))))
122
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
151
         (let ((result strs))
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
152
           ;; strs is not needed so we simply overwrite it
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
153
           (when type-conv-fns
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
154
             (setf result
277
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
155
                   (map 'vector #'funcall type-conv-fns result)))
122
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
156
           (when map-fns
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
157
             (setf result
277
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
158
                   (map 'vector #'funcall map-fns result)))
122
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
159
           result)))))
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
160
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
161
 (defun read-csv-stream (stream &key (header t) type-spec map-fns (delimiter *csv-separator*) (start 0) end)
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
162
   "Read from stream until eof and return a csv table.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
163
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
164
 A csv table is a vector of csv records.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
165
 A csv record is a vector of elements.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
166
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
167
 Type spec should be a list of type specifier (symbols).
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
168
 If the type specifier is nil or t, it will be treated as string.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
169
 If type-spec is nil (the default case), then all will be treated
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
170
 as string.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
171
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
172
 map-fns is a list of functions of one argument and output one result.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
173
 each function in it will be applied to the parsed element.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
174
 If any function in the list is nil or t, it equals to #'identity.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
175
 If map-fns is nil, then nothing will be applied.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
176
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
177
 start and end specifies how many elements per record will be included.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
178
 If start or end is negative, it counts from the end. -1 is the last element.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
179
 "
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
180
   (let ((type-conv-fns
277
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
181
           (when type-spec
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
182
             (macrolet ((make-num-specifier (specifier)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
183
                          `(lambda (s) (let ((s (parse-number-no-error s s)))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
184
                                         (if (numberp s) (funcall ,specifier s) s)))))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
185
               (map 'vector
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
186
                    (lambda (type)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
187
                      (ecase type
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
188
                        ((t nil string) #'identity)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
189
                        (number #'(lambda (s) (parse-number-no-error s s)))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
190
                        (float (make-num-specifier #'float))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
191
                        (single-float (make-num-specifier #'(lambda (s) (coerce s 'single-float))))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
192
                        (double-float (make-num-specifier #'(lambda (s) (coerce s 'double-float))))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
193
                        (integer (make-num-specifier #'round))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
194
                        (pathname #'pathname)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
195
                        (symbol #'intern)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
196
                        (keyword (lambda (s) (intern s :keyword)))))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
197
                    type-spec))))
122
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
198
         (map-fns
277
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
199
           (when map-fns
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
200
             (map 'vector
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
201
                  (lambda (fn)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
202
                    (cond ((or (eq fn t)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
203
                               (eq fn nil))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
204
                           #'identity)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
205
                          ((functionp fn)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
206
                           fn)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
207
                          ((and (symbolp fn)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
208
                                (not (keywordp fn)))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
209
                           (symbol-function fn))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
210
                          (t (error "~a is not a valid function specifier." fn))))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
211
                  map-fns)))
122
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
212
         (header
277
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
213
           (etypecase header
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
214
             (cons (coerce header 'vector))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
215
             (boolean (when header
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
216
                        (read-csv-line stream :delimiter delimiter))))))
122
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
217
     (loop for rec = (read-csv-line stream :type-conv-fns type-conv-fns :map-fns map-fns :delimiter delimiter
277
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
218
                                           :start start :end end)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
219
           while rec
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
220
           collect rec into result
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
221
           finally (return
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
222
                     (values
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
223
                      (coerce result 'vector)
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
224
                      header)))))
122
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
225
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
226
 (defun read-csv-file (filename &key (header t) type-spec map-fns (delimiter *csv-separator*) (external-format *csv-default-external-format*)
277
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
227
                                  (start 0) end)
122
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
228
   "Read from stream until eof and return a csv table.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
229
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
230
 A csv table is a vector of csv records.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
231
 A csv record is a vector of elements.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
232
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
233
 Type spec should be a list of type specifier (symbols).
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
234
 If the type specifier is nil or t, it will be treated as string.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
235
 If type-spec is nil (the default case), then all will be treated
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
236
 as string.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
237
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
238
 map-fns is a list of functions of one argument and output one result.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
239
 each function in it will be applied to the parsed element.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
240
 If any function in the list is nil or t, it equals to #'identity.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
241
 If map-fns is nil, then nothing will be applied.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
242
 https://cgit.gentoo.org/proj/lisp.git/tree/dev-lisp/cl-rsm-finance/cl-rsm-finance-1.1.ebuild?h=old-portage&id=e9b71910b0d4f22aeb66f14e158a2451f9955b0d
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
243
 external-format (default is shift-jis) is a valid AllegroCL external-format type.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
244
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
245
 OS is a set to eol-convention of the file stream.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
246
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
247
 start and end specifies how many elements per record will be included.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
248
 If start or end is negative, it counts from the end. -1 is the last element.
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
249
 "
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
250
   (with-open-file (f filename :external-format external-format)
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
251
     (read-csv-stream f :type-spec type-spec :map-fns map-fns
277
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
252
                        :delimiter delimiter
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
253
                        :start start :end end
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
254
                        :header header)))
122
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
255
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
256
 
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
257
 (defun read-csv-file-and-sort (filename sort-order &key (header t) (order :ascend) type-spec map-fns (delimiter *csv-separator*) (external-format *csv-default-external-format*))
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
258
   (let ((table (read-csv-file filename
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
259
                               :header header
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
260
                               :type-spec type-spec
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
261
                               :map-fns map-fns
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
262
                               :delimiter delimiter
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
263
                               :external-format external-format)))
4ba88cac5bc7 num/parse, added DAT system, net/fetch, time/local, refactored trees
ellis <ellis@rwest.io>
parents:
diff changeset
264
     (loop for i in (reverse sort-order)
277
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
265
           do (setf table
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
266
                    (stable-sort table (ecase order (:ascend #'string<=) (:descend #'string>=))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
267
                                 :key (lambda (rec) (aref rec i))))
10faf95f90dd stream and basic type upgrades. fixed some bugs and improved csv parsing
Richard Westhaver <ellis@rwest.io>
parents: 275
diff changeset
268
           finally (return table))))