1.1--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2+++ b/examples/db/mbdb.lisp	Thu Apr 11 18:58:35 2024 -0400
     1.3@@ -0,0 +1,48 @@
     1.4+;;; examples/mbdb.lisp --- MusicBrainz Database import and analysis
     1.5+
     1.6+;; This example show how to migrate a set of complex JSON objects to
     1.7+;; RocksDB using a dump from the MusicBrainz database
     1.8+;; (https://musicbrainz.org/). The files are hosted at
     1.9+;; https://packy.compiler.company/data/mbdump
    1.10+
    1.11+;;; Code:
    1.12+(defpackage :examples/mbdb
    1.13+  (:use :cl :std :dat/json :net/fetch :obj/id :rdb :cli/clap :obj/uuid)
    1.14+  (:import-from :log :info! :debug!)
    1.15+  (:import-from :rocksdb :load-rocksdb)
    1.16+  (:export :main))
    1.17+
    1.18+(in-package :examples/mbdb)
    1.19+
    1.20+(load-rocksdb t)
    1.21+
    1.22+(declaim (type pathname *mbdb-path*))
    1.23+(defvar *mbdb-path* #P"/tmp/mbdb/")
    1.24+(defvar *mbdb* (create-db *mbdb-path* :opts (default-rdb-opts)))
    1.25+
    1.26+(defvar *mbdump-base-url* "https://packy.compiler.company/data/mbdump/"
    1.27+  "Remote location of MusicBrainz JSON data files.")
    1.28+
    1.29+(defvar *mbdump-files*
    1.30+  (mapcar (lambda (f) (make-pathname :name f :type "json" :directory *mbdump-base-url*))
    1.31+          (list "area" "artist" "event" "instrument"
    1.32+                "label" "place" "recording" "release"
    1.33+                "release-group" "series" "work")))
    1.34+
    1.35+(defun extract-columns (obj)
    1.36+  "Extract fields from a JSON-OBJECT, returning a vector of
    1.37+  uninitialized column-families which can be created with CREATE-CFS.
    1.38+
    1.39+Returns multiple values: the list of columns, the id, and type-id if present."
    1.40+  (values
    1.41+   (mapcar #'car (json-object-members obj))
    1.42+   (make-uuid-from-string (json-getf obj "id"))
    1.43+   (when-let ((tid (json-getf obj "type-id")))
    1.44+     (make-uuid-from-string tid))))
    1.45+
    1.46+(defmain ()
    1.47+  (let ((*default-pathname-defaults* (ensure-directories-exist *mbdb-path* :verbose t)))
    1.48+    (log:info! "Welcome to MBDB")
    1.49+    (with-db (db *mbdb*)
    1.50+      (open-db db)
    1.51+      (close-db db))))