changelog shortlog graph tags branches changeset files revisions annotate raw help

Mercurial > demo / examples/db/mbdb.lisp

changeset 39: 1ef551e24009
child: 6b652d7d6663
author: Richard Westhaver <ellis@rwest.io>
date: Thu, 11 Apr 2024 18:58:35 -0400
permissions: -rw-r--r--
description: added musicbrainz db example 
     1 ;;; examples/mbdb.lisp --- MusicBrainz Database import and analysis
     2 
     3 ;; This example show how to migrate a set of complex JSON objects to
     4 ;; RocksDB using a dump from the MusicBrainz database
     5 ;; (https://musicbrainz.org/). The files are hosted at
     6 ;; https://packy.compiler.company/data/mbdump
     7 
     8 ;;; Code:
     9 (defpackage :examples/mbdb
    10   (:use :cl :std :dat/json :net/fetch :obj/id :rdb :cli/clap :obj/uuid)
    11   (:import-from :log :info! :debug!)
    12   (:import-from :rocksdb :load-rocksdb)
    13   (:export :main))
    14 
    15 (in-package :examples/mbdb)
    16 
    17 (load-rocksdb t)
    18 
    19 (declaim (type pathname *mbdb-path*))
    20 (defvar *mbdb-path* #P"/tmp/mbdb/")
    21 (defvar *mbdb* (create-db *mbdb-path* :opts (default-rdb-opts)))
    22 
    23 (defvar *mbdump-base-url* "https://packy.compiler.company/data/mbdump/"
    24   "Remote location of MusicBrainz JSON data files.")
    25 
    26 (defvar *mbdump-files*
    27   (mapcar (lambda (f) (make-pathname :name f :type "json" :directory *mbdump-base-url*))
    28           (list "area" "artist" "event" "instrument"
    29                 "label" "place" "recording" "release"
    30                 "release-group" "series" "work")))
    31 
    32 (defun extract-columns (obj)
    33   "Extract fields from a JSON-OBJECT, returning a vector of
    34   uninitialized column-families which can be created with CREATE-CFS.
    35 
    36 Returns multiple values: the list of columns, the id, and type-id if present."
    37   (values
    38    (mapcar #'car (json-object-members obj))
    39    (make-uuid-from-string (json-getf obj "id"))
    40    (when-let ((tid (json-getf obj "type-id")))
    41      (make-uuid-from-string tid))))
    42 
    43 (defmain ()
    44   (let ((*default-pathname-defaults* (ensure-directories-exist *mbdb-path* :verbose t)))
    45     (log:info! "Welcome to MBDB")
    46     (with-db (db *mbdb*)
    47       (open-db db)
    48       (close-db db))))