poc leveldb
This commit is contained in:
Ben
parent
46d7df8a6e
commit
2375a8f852
|
|
@ -0,0 +1,38 @@
|
|||
dist: bionic
|
||||
sudo: false
|
||||
|
||||
language: c
|
||||
|
||||
cache:
|
||||
directories:
|
||||
- "$HOME/.nimble"
|
||||
- "$HOME/.choosenim"
|
||||
|
||||
addons:
|
||||
apt:
|
||||
packages:
|
||||
- libsnappy1v5
|
||||
|
||||
matrix:
|
||||
fast_finish: true
|
||||
include:
|
||||
- os: linux
|
||||
env: CHANNEL=stable
|
||||
- os: linux
|
||||
env: CHANNEL=1.4.0
|
||||
|
||||
install:
|
||||
- |
|
||||
wget http://ftp.us.debian.org/debian/pool/main/l/leveldb/libleveldb1d_1.22-3_amd64.deb
|
||||
wget http://ftp.us.debian.org/debian/pool/main/l/leveldb/libleveldb-dev_1.22-3_amd64.deb
|
||||
sudo dpkg -i *.deb
|
||||
- |
|
||||
curl https://nim-lang.org/choosenim/init.sh -sSf > init.sh
|
||||
sh init.sh -y
|
||||
- export CHOOSENIM_NO_ANALYTICS=1
|
||||
- export PATH=$HOME/.nimble/bin:$PATH
|
||||
- echo "export PATH=~/.nimble/bin:$PATH" >> ~/.profile
|
||||
- yes | choosenim $CHANNEL
|
||||
|
||||
script:
|
||||
- nimble test
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
# leveldb.nim
|
||||
|
||||
[](https://zielmicha.github.io/leveldb.nim/)
|
||||
|
||||
A LevelDB wrapper for Nim in a Nim friendly way.
|
||||
|
||||
Create a database:
|
||||
```Nim
|
||||
import leveldb
|
||||
import options
|
||||
|
||||
var db = leveldb.open("/tmp/mydata")
|
||||
```
|
||||
|
||||
Read or modify the database content:
|
||||
```Nim
|
||||
assert db.getOrDefault("nothing", "") == ""
|
||||
|
||||
db.put("hello", "world")
|
||||
db.put("bin", "GIF89a\1\0")
|
||||
echo db.get("hello")
|
||||
assert db.get("hello").isSome()
|
||||
|
||||
var key, val = ""
|
||||
for key, val in db.iter():
|
||||
echo key, ": ", repr(val)
|
||||
|
||||
db.delete("hello")
|
||||
assert db.get("hello").isNone()
|
||||
```
|
||||
|
||||
Batch writes:
|
||||
```Nim
|
||||
let batch = newBatch()
|
||||
for i in 1..10:
|
||||
batch.put("key" & $i, $i)
|
||||
batch.delete("bin")
|
||||
db.write(batch)
|
||||
```
|
||||
|
||||
Iterate over subset of database content:
|
||||
```Nim
|
||||
for key, val in db.iterPrefix(prefix = "key1"):
|
||||
echo key, ": ", val
|
||||
for key, val in db.iter(seek = "key3", reverse = true):
|
||||
echo key, ": ", val
|
||||
|
||||
db.close()
|
||||
```
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
# Package
|
||||
|
||||
version = "0.4.1"
|
||||
author = "Michał Zieliński"
|
||||
description = "LevelDB wrapper for Nim"
|
||||
license = "MIT"
|
||||
srcDir = "src"
|
||||
installExt = @["nim"]
|
||||
bin = @["leveldbtool"]
|
||||
|
||||
# Dependencies
|
||||
|
||||
requires "nim >= 1.4.0"
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
import std/times
|
||||
import std/options
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/sqlite3_abi
|
||||
from pkg/stew/results as stewResults import isErr
|
||||
import pkg/upraises
|
||||
|
||||
import pkg/datastore
|
||||
|
||||
import ./src/leveldb
|
||||
|
||||
push: {.upraises: [].}
|
||||
|
||||
type
|
||||
LevelDbDatastore* = ref object of Datastore
|
||||
db: LevelDb
|
||||
|
||||
func toByteSeq(str: string): seq[byte] {.inline.} =
|
||||
@(str.toOpenArrayByte(0, str.high))
|
||||
|
||||
func toString(bytes: openArray[byte]): string {.inline.} =
|
||||
let length = bytes.len
|
||||
if length > 0:
|
||||
result = newString(length)
|
||||
copyMem(result.cstring, bytes[0].unsafeAddr, length)
|
||||
|
||||
method get*(self: LevelDbDatastore, key: Key): Future[?!seq[byte]] {.async, locks: "unknown".} =
|
||||
try:
|
||||
let str = self.db.get($key)
|
||||
if not str.isSome:
|
||||
return failure("Not some!")
|
||||
let bytes = toByteSeq(str.get())
|
||||
return success(bytes)
|
||||
except LevelDbException:
|
||||
return failure("exception get")
|
||||
|
||||
method put*(self: LevelDbDatastore, key: Key, data: seq[byte]): Future[?!void] {.async, locks: "unknown".} =
|
||||
try:
|
||||
let str = toString(data)
|
||||
self.db.put($key, str)
|
||||
return success()
|
||||
except LevelDbException:
|
||||
return failure("exception put")
|
||||
|
||||
proc new*(
|
||||
T: type LevelDbDatastore, dbName: string): ?!T =
|
||||
try:
|
||||
let db = leveldb.open(dbName)
|
||||
|
||||
success T(
|
||||
db: db
|
||||
)
|
||||
except LevelDbException:
|
||||
return failure("exception open")
|
||||
|
|
@ -0,0 +1,423 @@
|
|||
## A LevelDB_ wrapper for Nim in a Nim friendly way.
|
||||
##
|
||||
## LevelDB is a fast and simple key/value data storage library built
|
||||
## by Google that provides an ordered mapping from string keys to
|
||||
## string values.
|
||||
##
|
||||
## .. _LevelDB: https://github.com/google/leveldb
|
||||
##
|
||||
## Create a database:
|
||||
##
|
||||
## .. code-block:: Nim
|
||||
## import leveldb
|
||||
## import options
|
||||
##
|
||||
## var db = leveldb.open("/tmp/mydata")
|
||||
##
|
||||
## Read or modify the database content:
|
||||
##
|
||||
## .. code-block:: Nim
|
||||
##
|
||||
## assert db.getOrDefault("nothing", "") == ""
|
||||
##
|
||||
## db.put("hello", "world")
|
||||
## db.put("bin", "GIF89a\1\0")
|
||||
## echo db.get("hello")
|
||||
## assert db.get("hello").isSome()
|
||||
##
|
||||
## var key, val = ""
|
||||
## for key, val in db.iter():
|
||||
## echo key, ": ", repr(val)
|
||||
##
|
||||
## db.delete("hello")
|
||||
## assert db.get("hello").isNone()
|
||||
##
|
||||
## Batch writes:
|
||||
##
|
||||
## .. code-block:: Nim
|
||||
## let batch = newBatch()
|
||||
## for i in 1..10:
|
||||
## batch.put("key" & $i, $i)
|
||||
## batch.delete("bin")
|
||||
## db.write(batch)
|
||||
##
|
||||
## Iterate over subset of database content:
|
||||
##
|
||||
## .. code-block:: Nim
|
||||
## for key, val in db.iterPrefix(prefix = "key1"):
|
||||
## echo key, ": ", val
|
||||
## for key, val in db.iter(seek = "key3", reverse = true):
|
||||
## echo key, ": ", val
|
||||
##
|
||||
## db.close()
|
||||
|
||||
import options, os, strutils
|
||||
import leveldb/raw
|
||||
|
||||
type
|
||||
LevelDb* = ref object
|
||||
path*: string
|
||||
db: ptr leveldb_t
|
||||
cache: ptr leveldb_cache_t
|
||||
readOptions: ptr leveldb_readoptions_t
|
||||
syncWriteOptions: ptr leveldb_writeoptions_t
|
||||
asyncWriteOptions: ptr leveldb_writeoptions_t
|
||||
|
||||
LevelDbWriteBatch* = ref object
|
||||
## Write batches for bulk data modification.
|
||||
batch: ptr leveldb_writebatch_t
|
||||
|
||||
CompressionType* = enum
|
||||
## No compression or using Snappy_ algorithm (default).
|
||||
##
|
||||
## .. _Snappy: http://google.github.io/snappy/
|
||||
ctNoCompression = leveldb_no_compression,
|
||||
ctSnappyCompression = leveldb_snappy_compression
|
||||
|
||||
LevelDbException* = object of CatchableError
|
||||
|
||||
const
|
||||
version* = block:
|
||||
const configFile = "leveldb.nimble"
|
||||
const sourcePath = currentSourcePath()
|
||||
const parentConfig = sourcePath.parentDir.parentDir / configFile
|
||||
const localConfig = sourcePath.parentDir / configFile
|
||||
var content: string
|
||||
if fileExists(parentConfig):
|
||||
content = staticRead(parentConfig)
|
||||
else:
|
||||
content = staticRead(localConfig)
|
||||
var version_line: string
|
||||
for line in content.split("\L"):
|
||||
if line.startsWith("version"):
|
||||
version_line = line
|
||||
break
|
||||
let raw = version_line.split("=", maxsplit = 1)[1]
|
||||
raw.strip().strip(chars = {'"'})
|
||||
|
||||
levelDbTrue = uint8(1)
|
||||
levelDbFalse = uint8(0)
|
||||
|
||||
proc free(p: pointer) {.importc.}
|
||||
|
||||
proc checkError(errPtr: cstring) =
|
||||
if errPtr != nil:
|
||||
defer: free(errPtr)
|
||||
raise newException(LevelDbException, $errPtr)
|
||||
|
||||
proc getLibVersion*(): (int, int) =
|
||||
## Get the version of leveldb C library.
|
||||
result[0] = leveldb_major_version()
|
||||
result[1] = leveldb_minor_version()
|
||||
|
||||
proc close*(self: LevelDb) =
|
||||
## Closes the database.
|
||||
##
|
||||
## See also:
|
||||
## * `open proc <#open%2Cstring%2Cint%2Cint%2Cint>`_
|
||||
if self.db == nil:
|
||||
return
|
||||
leveldb_close(self.db)
|
||||
leveldb_writeoptions_destroy(self.syncWriteOptions)
|
||||
leveldb_writeoptions_destroy(self.asyncWriteOptions)
|
||||
leveldb_readoptions_destroy(self.readOptions)
|
||||
if self.cache != nil:
|
||||
leveldb_cache_destroy(self.cache)
|
||||
self.cache = nil
|
||||
self.db = nil
|
||||
|
||||
proc open*(path: string, create = true, reuse = true, paranoidChecks = true,
|
||||
compressionType = ctSnappyCompression,
|
||||
cacheCapacity = 0, blockSize = 4 * 1024, writeBufferSize = 4*1024*1024,
|
||||
maxOpenFiles = 1000, maxFileSize = 2 * 1024 * 1024,
|
||||
blockRestartInterval = 16): LevelDb =
|
||||
## Opens a database.
|
||||
##
|
||||
## Raises `LevelDbException` if corruption detected in the database.
|
||||
##
|
||||
## See also:
|
||||
## * `close proc <#close%2CLevelDb>`_
|
||||
new(result, close)
|
||||
|
||||
let options = leveldb_options_create()
|
||||
defer: leveldb_options_destroy(options)
|
||||
|
||||
result.syncWriteOptions = leveldb_writeoptions_create()
|
||||
leveldb_writeoptions_set_sync(result.syncWriteOptions, levelDbTrue)
|
||||
result.asyncWriteOptions = leveldb_writeoptions_create()
|
||||
leveldb_writeoptions_set_sync(result.asyncWriteOptions, levelDbFalse)
|
||||
result.readOptions = leveldb_readoptions_create()
|
||||
|
||||
if create:
|
||||
leveldb_options_set_create_if_missing(options, levelDbTrue)
|
||||
else:
|
||||
leveldb_options_set_create_if_missing(options, levelDbFalse)
|
||||
if reuse:
|
||||
leveldb_options_set_error_if_exists(options, levelDbFalse)
|
||||
else:
|
||||
leveldb_options_set_error_if_exists(options, levelDbTrue)
|
||||
if paranoidChecks:
|
||||
leveldb_options_set_paranoid_checks(options, levelDbTrue)
|
||||
else:
|
||||
leveldb_options_set_paranoid_checks(options, levelDbFalse)
|
||||
|
||||
leveldb_options_set_write_buffer_size(options, writeBufferSize.csize_t)
|
||||
leveldb_options_set_block_size(options, blockSize.csize_t)
|
||||
leveldb_options_set_max_open_files(options, cast[cint](maxOpenFiles))
|
||||
leveldb_options_set_max_file_size(options, maxFileSize.csize_t)
|
||||
leveldb_options_set_block_restart_interval(options,
|
||||
cast[cint](blockRestartInterval))
|
||||
leveldb_options_set_compression(options, cast[cint](compressionType.ord))
|
||||
|
||||
if cacheCapacity > 0:
|
||||
let cache = leveldb_cache_create_lru(cacheCapacity.csize_t)
|
||||
leveldb_options_set_cache(options, cache)
|
||||
result.cache = cache
|
||||
|
||||
var errPtr: cstring = nil
|
||||
result.path = path
|
||||
result.db = leveldb_open(options, path, addr errPtr)
|
||||
checkError(errPtr)
|
||||
|
||||
proc put*(self: LevelDb, key: string, value: string, sync = false) =
|
||||
## Set a `value` for the specified `key`.
|
||||
##
|
||||
## By default, `sync` is turned off, each write to leveldb is asynchronous.
|
||||
## Unless reboot, a crash of just the writing process will not cause any
|
||||
## loss since even when `sync` is false.
|
||||
##
|
||||
## See also:
|
||||
## * `put proc <#put%2CLevelDbWriteBatch%2Cstring%2Cstring>`_
|
||||
runnableExamples:
|
||||
let db = leveldb.open("/tmp/test")
|
||||
db.put("hello", "world")
|
||||
db.close()
|
||||
|
||||
assert self.db != nil
|
||||
var errPtr: cstring = nil
|
||||
let writeOptions = if sync: self.syncWriteOptions else: self.asyncWriteOptions
|
||||
leveldb_put(self.db, writeOptions,
|
||||
key, key.len.csize_t, value, value.len.csize_t, addr errPtr)
|
||||
checkError(errPtr)
|
||||
|
||||
proc newString(cstr: cstring, length: csize_t): string =
|
||||
if length > 0:
|
||||
result = newString(length)
|
||||
copyMem(unsafeAddr result[0], cstr, length)
|
||||
else:
|
||||
result = ""
|
||||
|
||||
proc get*(self: LevelDb, key: string): Option[string] =
|
||||
## Get the value for the specified `key`.
|
||||
##
|
||||
## See also:
|
||||
## * `getOrDefault proc <#getOrDefault%2CLevelDb%2Cstring>`_
|
||||
runnableExamples:
|
||||
let db = leveldb.open("/tmp/test")
|
||||
db.put("hello", "world")
|
||||
echo db.get("hello")
|
||||
db.close()
|
||||
|
||||
var size: csize_t
|
||||
var errPtr: cstring = nil
|
||||
let s = leveldb_get(self.db, self.readOptions, key, key.len.csize_t, addr size, addr errPtr)
|
||||
checkError(errPtr)
|
||||
|
||||
if s == nil:
|
||||
result = none(string)
|
||||
else:
|
||||
result = some(newString(s, size))
|
||||
free(s)
|
||||
|
||||
proc getOrDefault*(self: LevelDb, key: string, default = ""): string =
|
||||
## Get the value for the specified `key`, or `default` if no value was set.
|
||||
##
|
||||
## See also:
|
||||
## * `get proc <#get%2CLevelDb%2Cstring>`_
|
||||
runnableExamples:
|
||||
let db = leveldb.open("/tmp/test")
|
||||
doAssert db.getOrDefault("what?", "nothing") == "nothing"
|
||||
db.close()
|
||||
|
||||
let val = self.get(key)
|
||||
if val.isNone():
|
||||
result = default
|
||||
else:
|
||||
result = val.get()
|
||||
|
||||
proc delete*(self: LevelDb, key: string, sync = false) =
|
||||
## Delete the key/value pair for the specified key.
|
||||
##
|
||||
## See also:
|
||||
## * `delete proc <#delete%2CLevelDbWriteBatch%2Cstring>`_
|
||||
var errPtr: cstring = nil
|
||||
let writeOptions = if sync: self.syncWriteOptions else: self.asyncWriteOptions
|
||||
leveldb_delete(self.db, writeOptions, key, key.len.csize_t, addr errPtr)
|
||||
checkError(errPtr)
|
||||
|
||||
proc destroy*(self: LevelDbWriteBatch) =
|
||||
## Destroys this batch.
|
||||
##
|
||||
## See also:
|
||||
## * `newBatch proc <#newBatch>`_
|
||||
if self.batch == nil:
|
||||
return
|
||||
leveldb_writebatch_destroy(self.batch)
|
||||
self.batch = nil
|
||||
|
||||
proc newBatch*(): LevelDbWriteBatch =
|
||||
## Creates a new database write batch.
|
||||
##
|
||||
## See also:
|
||||
## * `write proc <#write%2CLevelDb%2CLevelDbWriteBatch>`_
|
||||
## * `put proc <#put%2CLevelDbWriteBatch%2Cstring%2Cstring>`_
|
||||
## * `delete proc <#delete%2CLevelDbWriteBatch%2Cstring>`_
|
||||
## * `append proc <#append%2CLevelDbWriteBatch%2CLevelDbWriteBatch>`_
|
||||
## * `clear proc <#clear%2CLevelDbWriteBatch>`_
|
||||
## * `destroy proc <#destroy%2CLevelDbWriteBatch>`_
|
||||
|
||||
runnableExamples:
|
||||
let db = leveldb.open("/tmp/test")
|
||||
let batch = newBatch()
|
||||
for i in 1..10:
|
||||
batch.put("key" & $i, $i)
|
||||
batch.delete("another")
|
||||
db.write(batch)
|
||||
db.close()
|
||||
|
||||
new(result, destroy)
|
||||
result.batch = leveldb_writebatch_create()
|
||||
|
||||
proc put*(self: LevelDbWriteBatch, key: string, value: string, sync = false) =
|
||||
## Set a `value` for the specified `key`.
|
||||
## Same as `put <#put%2CLevelDb%2Cstring%2Cstring>`_ but operates on the
|
||||
## write batch instead.
|
||||
##
|
||||
## See also:
|
||||
## * `put proc <#put%2CLevelDb%2Cstring%2Cstring>`_
|
||||
## * `newBatch proc <#newBatch>`_
|
||||
leveldb_writebatch_put(self.batch, key, key.len.csize_t, value, value.len.csize_t)
|
||||
|
||||
proc append*(self, source: LevelDbWriteBatch) =
|
||||
## Merges the `source` batch into this batch.
|
||||
##
|
||||
## See also:
|
||||
## * `newBatch proc <#newBatch>`_
|
||||
leveldb_writebatch_append(self.batch, source.batch)
|
||||
|
||||
proc delete*(self: LevelDbWriteBatch, key: string) =
|
||||
## Delete the key/value pair for the specified `key`.
|
||||
## Same as `delete <#delete%2CLevelDb%2Cstring>`_ but operates on the
|
||||
## write batch instead.
|
||||
##
|
||||
## See also:
|
||||
## * `delete proc <#delete%2CLevelDb%2Cstring>`_
|
||||
## * `newBatch proc <#newBatch>`_
|
||||
leveldb_writebatch_delete(self.batch, key, key.len.csize_t)
|
||||
|
||||
proc clear*(self: LevelDbWriteBatch) =
|
||||
## Clear all updates buffered in this batch.
|
||||
##
|
||||
## See also:
|
||||
## * `newBatch proc <#newBatch>`_
|
||||
leveldb_writebatch_clear(self.batch)
|
||||
|
||||
proc write*(self: LevelDb, batch: LevelDbWriteBatch) =
|
||||
## Write apply the given `batch` to the database.
|
||||
##
|
||||
## See also:
|
||||
## * `newBatch proc <#newBatch>`_
|
||||
var errPtr: cstring = nil
|
||||
leveldb_write(self.db, self.syncWriteOptions, batch.batch, addr errPtr)
|
||||
checkError(errPtr)
|
||||
|
||||
proc getIterData(iterPtr: ptr leveldb_iterator_t): (string, string) =
|
||||
var len: csize_t
|
||||
var str: cstring
|
||||
|
||||
str = leveldb_iter_key(iterPtr, addr len)
|
||||
result[0] = newString(str, len)
|
||||
|
||||
str = leveldb_iter_value(iterPtr, addr len)
|
||||
result[1] = newString(str, len)
|
||||
|
||||
iterator iter*(self: LevelDb, seek: string = "", reverse: bool = false): (
|
||||
string, string) =
|
||||
## Iterate all key/value pairs in the database from the first one or
|
||||
## the specified key `seek`.
|
||||
## By default, the ordering will be lexicographic byte-wise ordering
|
||||
## with leveldb builtin comparator, unless `reverse` set to `true`.
|
||||
##
|
||||
## See also:
|
||||
## * `iterPrefix iterator <#iterPrefix.i%2CLevelDb%2Cstring>`_
|
||||
## * `iterRange iterator <#iterRange.i%2CLevelDb%2Cstring%2Cstring>`_
|
||||
var iterPtr = leveldb_create_iterator(self.db, self.readOptions)
|
||||
defer: leveldb_iter_destroy(iterPtr)
|
||||
|
||||
if seek.len > 0:
|
||||
leveldb_iter_seek(iterPtr, seek, seek.len.csize_t)
|
||||
else:
|
||||
if reverse:
|
||||
leveldb_iter_seek_to_last(iterPtr)
|
||||
else:
|
||||
leveldb_iter_seek_to_first(iterPtr)
|
||||
|
||||
while true:
|
||||
if leveldb_iter_valid(iterPtr) == levelDbFalse:
|
||||
break
|
||||
|
||||
var (key, value) = getIterData(iterPtr)
|
||||
var err: cstring = nil
|
||||
leveldb_iter_get_error(iterPtr, addr err)
|
||||
checkError(err)
|
||||
yield (key, value)
|
||||
|
||||
if reverse:
|
||||
leveldb_iter_prev(iterPtr)
|
||||
else:
|
||||
leveldb_iter_next(iterPtr)
|
||||
|
||||
iterator iterPrefix*(self: LevelDb, prefix: string): (string, string) =
|
||||
## Iterate subset key/value pairs in the database with a particular `prefix`.
|
||||
##
|
||||
## See also:
|
||||
## * `iter iterator <#iter.i%2CLevelDb%2Cstring%2Cbool>`_
|
||||
## * `iterRange iterator <#iterRange.i%2CLevelDb%2Cstring%2Cstring>`_
|
||||
for key, value in iter(self, prefix, reverse = false):
|
||||
if key.startsWith(prefix):
|
||||
yield (key, value)
|
||||
else:
|
||||
break
|
||||
|
||||
iterator iterRange*(self: LevelDb, start, limit: string): (string, string) =
|
||||
## Yields all key/value pairs between the `start` and `limit` keys
|
||||
## (inclusive) in the database.
|
||||
##
|
||||
## See also:
|
||||
## * `iter iterator <#iter.i%2CLevelDb%2Cstring%2Cbool>`_
|
||||
## * `iterPrefix iterator <#iterPrefix.i%2CLevelDb%2Cstring>`_
|
||||
let reverse: bool = limit < start
|
||||
for key, value in iter(self, start, reverse = reverse):
|
||||
if reverse:
|
||||
if key < limit:
|
||||
break
|
||||
else:
|
||||
if key > limit:
|
||||
break
|
||||
yield (key, value)
|
||||
|
||||
proc removeDb*(name: string) =
|
||||
## Remove the database `name`.
|
||||
var err: cstring = nil
|
||||
let options = leveldb_options_create()
|
||||
leveldb_destroy_db(options, name, addr err)
|
||||
checkError(err)
|
||||
|
||||
proc repairDb*(name: string) =
|
||||
## Repairs the corrupted database `name`.
|
||||
let options = leveldb_options_create()
|
||||
leveldb_options_set_create_if_missing(options, levelDbFalse)
|
||||
leveldb_options_set_error_if_exists(options, levelDbFalse)
|
||||
var errPtr: cstring = nil
|
||||
leveldb_repair_db(options, name, addr errPtr)
|
||||
checkError(errPtr)
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
# Package
|
||||
|
||||
version = "0.4.1"
|
||||
author = "Michał Zieliński"
|
||||
description = "LevelDB wrapper for Nim"
|
||||
license = "MIT"
|
||||
srcDir = "src"
|
||||
installExt = @["nim"]
|
||||
bin = @["leveldbtool"]
|
||||
|
||||
# Dependencies
|
||||
|
||||
requires "nim >= 1.4.0"
|
||||
|
|
@ -0,0 +1,199 @@
|
|||
## Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
## Use of this source code is governed by a BSD-style license that can be
|
||||
## found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
##
|
||||
## C bindings for leveldb. May be useful as a stable ABI that can be
|
||||
## used by programs that keep leveldb in a shared library, or for
|
||||
## a JNI api.
|
||||
##
|
||||
## Does not support:
|
||||
## . getters for the option types
|
||||
## . custom comparators that implement key shortening
|
||||
## . custom iter, db, env, cache implementations using just the C bindings
|
||||
##
|
||||
## Some conventions:
|
||||
##
|
||||
## (1) We expose just opaque struct pointers and functions to clients.
|
||||
## This allows us to change internal representations without having to
|
||||
## recompile clients.
|
||||
##
|
||||
## (2) For simplicity, there is no equivalent to the Slice type. Instead,
|
||||
## the caller has to pass the pointer and length as separate
|
||||
## arguments.
|
||||
##
|
||||
## (3) Errors are represented by a null-terminated c string. NULL
|
||||
## means no error. All operations that can raise an error are passed
|
||||
## a "char** errptr" as the last argument. One of the following must
|
||||
## be true on entry:
|
||||
## errptr == NULL
|
||||
## errptr points to a malloc()ed null-terminated error message
|
||||
## (On Windows, \*errptr must have been malloc()-ed by this library.)
|
||||
## On success, a leveldb routine leaves \*errptr unchanged.
|
||||
## On failure, leveldb frees the old value of \*errptr and
|
||||
## set \*errptr to a malloc()ed error message.
|
||||
##
|
||||
## (4) Bools have the type uint8_t (0 == false; rest == true)
|
||||
##
|
||||
## (5) All of the pointer arguments must be non-NULL.
|
||||
##
|
||||
|
||||
{.passl: "-lleveldb".}
|
||||
|
||||
## # Exported types
|
||||
|
||||
type
|
||||
leveldb_options_t* = object
|
||||
leveldb_writeoptions_t* = object
|
||||
leveldb_readoptions_t* = object
|
||||
leveldb_writebatch_t* = object
|
||||
leveldb_iterator_t* = object
|
||||
leveldb_snapshot_t* = object
|
||||
leveldb_comparator_t* = object
|
||||
leveldb_filterpolicy_t* = object
|
||||
leveldb_env_t* = object
|
||||
leveldb_logger_t* = object
|
||||
leveldb_cache_t* = object
|
||||
leveldb_t* = object
|
||||
|
||||
## DB operations
|
||||
|
||||
proc leveldb_open*(options: ptr leveldb_options_t; name: cstring; errptr: ptr cstring): ptr leveldb_t {.importc.}
|
||||
proc leveldb_close*(db: ptr leveldb_t) {.importc.}
|
||||
proc leveldb_put*(db: ptr leveldb_t; options: ptr leveldb_writeoptions_t; key: cstring;
|
||||
keylen: csize_t; val: cstring; vallen: csize_t; errptr: ptr cstring) {.importc.}
|
||||
proc leveldb_delete*(db: ptr leveldb_t; options: ptr leveldb_writeoptions_t;
|
||||
key: cstring; keylen: csize_t; errptr: ptr cstring) {.importc.}
|
||||
proc leveldb_write*(db: ptr leveldb_t; options: ptr leveldb_writeoptions_t;
|
||||
batch: ptr leveldb_writebatch_t; errptr: ptr cstring) {.importc.}
|
||||
## Returns NULL if not found. A malloc()ed array otherwise.
|
||||
## Stores the length of the array in \*vallen.
|
||||
|
||||
proc leveldb_get*(db: ptr leveldb_t; options: ptr leveldb_readoptions_t; key: cstring;
|
||||
keylen: csize_t; vallen: ptr csize_t; errptr: ptr cstring): cstring {.importc.}
|
||||
proc leveldb_create_iterator*(db: ptr leveldb_t; options: ptr leveldb_readoptions_t): ptr leveldb_iterator_t {.importc.}
|
||||
proc leveldb_create_snapshot*(db: ptr leveldb_t): ptr leveldb_snapshot_t {.importc.}
|
||||
proc leveldb_release_snapshot*(db: ptr leveldb_t; snapshot: ptr leveldb_snapshot_t) {.importc.}
|
||||
## Returns NULL if property name is unknown.
|
||||
## Else returns a pointer to a malloc()-ed null-terminated value.
|
||||
|
||||
proc leveldb_property_value*(db: ptr leveldb_t; propname: cstring): cstring {.importc.}
|
||||
proc leveldb_approximate_sizes*(db: ptr leveldb_t; num_ranges: cint;
|
||||
range_start_key: ptr cstring;
|
||||
range_start_key_len: ptr csize_t;
|
||||
range_limit_key: ptr cstring;
|
||||
range_limit_key_len: ptr csize_t; sizes: ptr uint64) {.importc.}
|
||||
proc leveldb_compact_range*(db: ptr leveldb_t; start_key: cstring;
|
||||
start_key_len: csize_t; limit_key: cstring;
|
||||
limit_key_len: csize_t) {.importc.}
|
||||
## Management operations
|
||||
|
||||
proc leveldb_destroy_db*(options: ptr leveldb_options_t; name: cstring;
|
||||
errptr: ptr cstring) {.importc.}
|
||||
proc leveldb_repair_db*(options: ptr leveldb_options_t; name: cstring;
|
||||
errptr: ptr cstring) {.importc.}
|
||||
## Iterator
|
||||
|
||||
proc leveldb_iter_destroy*(a1: ptr leveldb_iterator_t) {.importc.}
|
||||
proc leveldb_iter_valid*(a1: ptr leveldb_iterator_t): uint8 {.importc.}
|
||||
proc leveldb_iter_seek_to_first*(a1: ptr leveldb_iterator_t) {.importc.}
|
||||
proc leveldb_iter_seek_to_last*(a1: ptr leveldb_iterator_t) {.importc.}
|
||||
proc leveldb_iter_seek*(a1: ptr leveldb_iterator_t; k: cstring; klen: csize_t) {.importc.}
|
||||
proc leveldb_iter_next*(a1: ptr leveldb_iterator_t) {.importc.}
|
||||
proc leveldb_iter_prev*(a1: ptr leveldb_iterator_t) {.importc.}
|
||||
proc leveldb_iter_key*(a1: ptr leveldb_iterator_t; klen: ptr csize_t): cstring {.importc.}
|
||||
proc leveldb_iter_value*(a1: ptr leveldb_iterator_t; vlen: ptr csize_t): cstring {.importc.}
|
||||
proc leveldb_iter_get_error*(a1: ptr leveldb_iterator_t; errptr: ptr cstring) {.importc.}
|
||||
## Write batch
|
||||
|
||||
proc leveldb_writebatch_create*(): ptr leveldb_writebatch_t {.importc.}
|
||||
proc leveldb_writebatch_destroy*(a1: ptr leveldb_writebatch_t) {.importc.}
|
||||
proc leveldb_writebatch_clear*(a1: ptr leveldb_writebatch_t) {.importc.}
|
||||
proc leveldb_writebatch_put*(a1: ptr leveldb_writebatch_t; key: cstring; klen: csize_t;
|
||||
val: cstring; vlen: csize_t) {.importc.}
|
||||
proc leveldb_writebatch_delete*(a1: ptr leveldb_writebatch_t; key: cstring;
|
||||
klen: csize_t) {.importc.}
|
||||
proc leveldb_writebatch_iterate*(a1: ptr leveldb_writebatch_t; state: pointer; put: proc (
|
||||
a1: pointer; k: cstring; klen: csize_t; v: cstring; vlen: csize_t); deleted: proc (
|
||||
a1: pointer; k: cstring; klen: csize_t)) {.importc.}
|
||||
proc leveldb_writebatch_append*(destination: ptr leveldb_writebatch_t;
|
||||
source: ptr leveldb_writebatch_t) {.importc.}
|
||||
## Options
|
||||
|
||||
proc leveldb_options_create*(): ptr leveldb_options_t {.importc.}
|
||||
proc leveldb_options_destroy*(a1: ptr leveldb_options_t) {.importc.}
|
||||
proc leveldb_options_set_comparator*(a1: ptr leveldb_options_t;
|
||||
a2: ptr leveldb_comparator_t) {.importc.}
|
||||
proc leveldb_options_set_filter_policy*(a1: ptr leveldb_options_t;
|
||||
a2: ptr leveldb_filterpolicy_t) {.importc.}
|
||||
proc leveldb_options_set_create_if_missing*(a1: ptr leveldb_options_t; a2: uint8) {.importc.}
|
||||
proc leveldb_options_set_error_if_exists*(a1: ptr leveldb_options_t; a2: uint8) {.importc.}
|
||||
proc leveldb_options_set_paranoid_checks*(a1: ptr leveldb_options_t; a2: uint8) {.importc.}
|
||||
proc leveldb_options_set_env*(a1: ptr leveldb_options_t; a2: ptr leveldb_env_t) {.importc.}
|
||||
proc leveldb_options_set_info_log*(a1: ptr leveldb_options_t;
|
||||
a2: ptr leveldb_logger_t) {.importc.}
|
||||
proc leveldb_options_set_write_buffer_size*(a1: ptr leveldb_options_t; a2: csize_t) {.importc.}
|
||||
proc leveldb_options_set_max_open_files*(a1: ptr leveldb_options_t; a2: cint) {.importc.}
|
||||
proc leveldb_options_set_cache*(a1: ptr leveldb_options_t; a2: ptr leveldb_cache_t) {.importc.}
|
||||
proc leveldb_options_set_block_size*(a1: ptr leveldb_options_t; a2: csize_t) {.importc.}
|
||||
proc leveldb_options_set_block_restart_interval*(a1: ptr leveldb_options_t; a2: cint) {.importc.}
|
||||
proc leveldb_options_set_max_file_size*(a1: ptr leveldb_options_t; a2: csize_t) {.importc.}
|
||||
const
|
||||
leveldb_no_compression* = 0
|
||||
leveldb_snappy_compression* = 1
|
||||
|
||||
proc leveldb_options_set_compression*(a1: ptr leveldb_options_t; a2: cint) {.importc.}
|
||||
## Comparator
|
||||
|
||||
proc leveldb_comparator_create*(state: pointer; destructor: proc (a1: pointer); compare: proc (
|
||||
a1: pointer; a: cstring; alen: csize_t; b: cstring; blen: csize_t): cint;
|
||||
name: proc (a1: pointer): cstring): ptr leveldb_comparator_t {.importc.}
|
||||
proc leveldb_comparator_destroy*(a1: ptr leveldb_comparator_t) {.importc.}
|
||||
## Filter policy
|
||||
|
||||
proc leveldb_filterpolicy_create*(state: pointer; destructor: proc (a1: pointer);
|
||||
create_filter: proc (a1: pointer; key_array: ptr cstring;
|
||||
key_length_array: ptr csize_t; num_keys: cint;
|
||||
filter_length: ptr csize_t): cstring; key_may_match: proc (
|
||||
a1: pointer; key: cstring; length: csize_t; filter: cstring; filter_length: csize_t): uint8;
|
||||
name: proc (a1: pointer): cstring): ptr leveldb_filterpolicy_t {.importc.}
|
||||
proc leveldb_filterpolicy_destroy*(a1: ptr leveldb_filterpolicy_t) {.importc.}
|
||||
proc leveldb_filterpolicy_create_bloom*(bits_per_key: cint): ptr leveldb_filterpolicy_t {.importc.}
|
||||
## Read options
|
||||
|
||||
proc leveldb_readoptions_create*(): ptr leveldb_readoptions_t {.importc.}
|
||||
proc leveldb_readoptions_destroy*(a1: ptr leveldb_readoptions_t) {.importc.}
|
||||
proc leveldb_readoptions_set_verify_checksums*(a1: ptr leveldb_readoptions_t;
|
||||
a2: uint8) {.importc.}
|
||||
proc leveldb_readoptions_set_fill_cache*(a1: ptr leveldb_readoptions_t; a2: uint8) {.importc.}
|
||||
proc leveldb_readoptions_set_snapshot*(a1: ptr leveldb_readoptions_t;
|
||||
a2: ptr leveldb_snapshot_t) {.importc.}
|
||||
## Write options
|
||||
|
||||
proc leveldb_writeoptions_create*(): ptr leveldb_writeoptions_t {.importc.}
|
||||
proc leveldb_writeoptions_destroy*(a1: ptr leveldb_writeoptions_t) {.importc.}
|
||||
proc leveldb_writeoptions_set_sync*(a1: ptr leveldb_writeoptions_t; a2: uint8) {.importc.}
|
||||
## Cache
|
||||
|
||||
proc leveldb_cache_create_lru*(capacity: csize_t): ptr leveldb_cache_t {.importc.}
|
||||
proc leveldb_cache_destroy*(cache: ptr leveldb_cache_t) {.importc.}
|
||||
## Env
|
||||
|
||||
proc leveldb_create_default_env*(): ptr leveldb_env_t {.importc.}
|
||||
proc leveldb_env_destroy*(a1: ptr leveldb_env_t) {.importc.}
|
||||
## If not NULL, the returned buffer must be released using leveldb_free().
|
||||
|
||||
proc leveldb_env_get_test_directory*(a1: ptr leveldb_env_t): cstring {.importc.}
|
||||
## Utility
|
||||
## Calls free(ptr).
|
||||
## REQUIRES: ptr was malloc()-ed and returned by one of the routines
|
||||
## in this file. Note that in certain cases (typically on Windows), you
|
||||
## may need to call this routine instead of free(ptr) to dispose of
|
||||
## malloc()-ed memory returned by this library.
|
||||
|
||||
proc leveldb_free*(`ptr`: pointer) {.importc.}
|
||||
## Return the major version number for this release.
|
||||
|
||||
proc leveldb_major_version*(): cint {.importc.}
|
||||
## Return the minor version number for this release.
|
||||
|
||||
proc leveldb_minor_version*(): cint {.importc.}
|
||||
|
|
@ -0,0 +1,129 @@
|
|||
import options, os, strutils
|
||||
import leveldb
|
||||
|
||||
proc tool() =
|
||||
proc usage() =
|
||||
echo "LevelDB client"
|
||||
echo ""
|
||||
echo "Usage:"
|
||||
echo " leveldb [-d <db_path>] create"
|
||||
echo " leveldb [-d <db_path>] get <key> [-x | --hex]"
|
||||
echo " leveldb [-d <db_path>] put <key> <value> [-x | --hex]"
|
||||
echo " leveldb [-d <db_path>] list [-x | --hex]"
|
||||
echo " leveldb [-d <db_path>] keys"
|
||||
echo " leveldb [-d <db_path>] delete <key>"
|
||||
echo " leveldb [-d <db_path>] repair"
|
||||
echo " leveldb -h | --help"
|
||||
echo " leveldb -v | --version"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " -d --database Database path"
|
||||
echo " -x --hex binary value in uppercase hex"
|
||||
echo " -h --help Show this screen"
|
||||
echo " -v --version Show version"
|
||||
quit()
|
||||
|
||||
var args = commandLineParams()
|
||||
|
||||
if "-h" in args or "--help" in args or len(args) == 0:
|
||||
usage()
|
||||
|
||||
if "-v" in args or "--version" in args:
|
||||
echo "leveldb.nim ", version
|
||||
let (major, minor) = getLibVersion()
|
||||
echo "leveldb ", major, ".", minor
|
||||
quit()
|
||||
|
||||
proc findArg(s: seq[string], item: string): int =
|
||||
result = find(s, item)
|
||||
let stop = find(s, "--")
|
||||
if stop >= 0 and stop <= result:
|
||||
result = -1
|
||||
|
||||
var dbPath = "./"
|
||||
var i = findArg(args, "-d")
|
||||
var j = findArg(args, "--database")
|
||||
if i >= 0 and j >= 0:
|
||||
quit("Please specify database path one time only.")
|
||||
i = max(i, j)
|
||||
if i >= 0:
|
||||
if (i + 1) < len(args):
|
||||
dbPath = args[i+1]
|
||||
args.delete(i+1)
|
||||
args.delete(i)
|
||||
else:
|
||||
quit("Please specify database path.")
|
||||
|
||||
var hex = false
|
||||
i = findArg(args, "-x")
|
||||
j = findArg(args, "--hex")
|
||||
if i >= 0:
|
||||
hex = true
|
||||
args.delete(i)
|
||||
if j >= 0:
|
||||
hex = true
|
||||
args.delete(j)
|
||||
|
||||
# drop stop word
|
||||
if "--" in args:
|
||||
args.delete(args.find("--"))
|
||||
|
||||
if len(args) == 0:
|
||||
usage()
|
||||
|
||||
proc checkCommand(args: seq[string], requires: int) =
|
||||
if len(args) < requires + 1:
|
||||
quit("Command " & args[0] & " requires at least " & $(requires) & " arguments.")
|
||||
|
||||
|
||||
var db: LevelDb
|
||||
var key, value: string
|
||||
if args[0] == "create":
|
||||
db = leveldb.open(dbPath)
|
||||
db.close()
|
||||
elif args[0] == "get":
|
||||
checkCommand(args, 1)
|
||||
db = leveldb.open(dbPath)
|
||||
key = args[1]
|
||||
let val = db.get(key)
|
||||
if val.isNone():
|
||||
quit()
|
||||
else:
|
||||
if hex:
|
||||
echo val.get().toHex()
|
||||
else:
|
||||
echo val.get()
|
||||
db.close()
|
||||
elif args[0] == "put":
|
||||
checkCommand(args, 2)
|
||||
db = leveldb.open(dbPath)
|
||||
key = args[1]
|
||||
value = args[2]
|
||||
if hex:
|
||||
value = parseHexStr(value)
|
||||
db.put(key, value)
|
||||
db.close()
|
||||
elif args[0] == "list":
|
||||
db = leveldb.open(dbPath)
|
||||
for key, value in db.iter():
|
||||
if hex:
|
||||
echo key, " ", value.toHex()
|
||||
else:
|
||||
echo key, " ", value
|
||||
db.close()
|
||||
elif args[0] == "keys":
|
||||
db = leveldb.open(dbPath)
|
||||
for key, value in db.iter():
|
||||
echo key
|
||||
db.close()
|
||||
elif args[0] == "delete":
|
||||
checkCommand(args, 1)
|
||||
db = leveldb.open(dbPath)
|
||||
key = args[1]
|
||||
db.delete(key)
|
||||
db.close()
|
||||
elif args[0] == "repair":
|
||||
repairDb(dbPath)
|
||||
|
||||
when isMainModule:
|
||||
tool()
|
||||
|
|
@ -0,0 +1 @@
|
|||
switch("path", "$projectDir/../src")
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
# Package
|
||||
|
||||
version = "0.1.0"
|
||||
author = "Xie Yanbo"
|
||||
description = "A new awesome nimble package"
|
||||
license = "MIT"
|
||||
srcDir = "src"
|
||||
bin = @["packagetest"]
|
||||
|
||||
|
||||
|
||||
# Dependencies
|
||||
|
||||
requires "nim >= 0.18.0"
|
||||
requires "leveldb"
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
import options
|
||||
import leveldb
|
||||
|
||||
when isMainModule:
|
||||
let db = leveldb.open("/tmp/testleveldb/tooldb")
|
||||
db.put("hello", "world")
|
||||
let val = db.get("hello")
|
||||
if val.isSome() and val.get() == "world":
|
||||
echo "leveldb works."
|
||||
db.close()
|
||||
|
|
@ -0,0 +1,263 @@
|
|||
import unittest, options, os, osproc, sequtils, strutils
|
||||
import leveldb, leveldb/raw
|
||||
|
||||
const
|
||||
tmpDir = "/tmp/testleveldb"
|
||||
tmpNimbleDir = tmpDir / "nimble"
|
||||
tmpDbDir = tmpDir / "testdb"
|
||||
|
||||
template cd*(dir: string, body: untyped) =
|
||||
## Sets the current dir to ``dir``, executes ``body`` and restores the
|
||||
## previous working dir.
|
||||
block:
|
||||
let lastDir = getCurrentDir()
|
||||
setCurrentDir(dir)
|
||||
body
|
||||
setCurrentDir(lastDir)
|
||||
|
||||
proc execNimble(args: varargs[string]): tuple[output: string, exitCode: int] =
|
||||
var quotedArgs = @args
|
||||
quotedArgs.insert("-y")
|
||||
quotedArgs.insert("--nimbleDir:" & tmpNimbleDir)
|
||||
quotedArgs.insert("nimble")
|
||||
quotedArgs = quotedArgs.map(proc (x: string): string = "\"" & x & "\"")
|
||||
|
||||
let cmd = quotedArgs.join(" ")
|
||||
result = execCmdEx(cmd)
|
||||
checkpoint(cmd)
|
||||
checkpoint(result.output)
|
||||
|
||||
proc execTool(args: varargs[string]): tuple[output: string, exitCode: int] =
|
||||
var quotedArgs = @args
|
||||
quotedArgs.insert(tmpDbDir)
|
||||
quotedArgs.insert("--database")
|
||||
quotedArgs.insert(tmpNimbleDir / "bin" / "leveldbtool")
|
||||
quotedArgs = quotedArgs.map(proc (x: string): string = "\"" & x & "\"")
|
||||
|
||||
if not dirExists(tmpDbDir):
|
||||
createDir(tmpDbDir)
|
||||
|
||||
let cmd = quotedArgs.join(" ")
|
||||
result = execCmdEx(cmd)
|
||||
checkpoint(cmd)
|
||||
checkpoint(result.output)
|
||||
|
||||
suite "leveldb":
|
||||
|
||||
setup:
|
||||
let env = leveldb_create_default_env()
|
||||
let dbName = $(leveldb_env_get_test_directory(env))
|
||||
let db = leveldb.open(dbName)
|
||||
|
||||
teardown:
|
||||
db.close()
|
||||
removeDb(dbName)
|
||||
|
||||
test "version":
|
||||
let (major, minor) = getLibVersion()
|
||||
check(major > 0)
|
||||
check(minor > 0)
|
||||
|
||||
test "get nothing":
|
||||
check(db.get("nothing") == none(string))
|
||||
|
||||
test "put and get":
|
||||
db.put("hello", "world")
|
||||
check(db.get("hello") == some("world"))
|
||||
|
||||
test "get or default":
|
||||
check(db.getOrDefault("nothing", "yes") == "yes")
|
||||
|
||||
test "delete":
|
||||
db.put("hello", "world")
|
||||
db.delete("hello")
|
||||
check(db.get("hello") == none(string))
|
||||
|
||||
test "get value with 0x00":
|
||||
db.put("\0key", "\0ff")
|
||||
check(db.get("\0key") == some("\0ff"))
|
||||
|
||||
test "get empty value":
|
||||
db.put("a", "")
|
||||
check(db.get("a") == some(""))
|
||||
|
||||
test "get empty key":
|
||||
db.put("", "a")
|
||||
check(db.get("") == some("a"))
|
||||
|
||||
proc initData(db: LevelDb) =
|
||||
db.put("aa", "1")
|
||||
db.put("ba", "2")
|
||||
db.put("bb", "3")
|
||||
|
||||
test "iter":
|
||||
initData(db)
|
||||
check(toSeq(db.iter()) == @[("aa", "1"), ("ba", "2"), ("bb", "3")])
|
||||
|
||||
test "iter reverse":
|
||||
initData(db)
|
||||
check(toSeq(db.iter(reverse = true)) ==
|
||||
@[("bb", "3"), ("ba", "2"), ("aa", "1")])
|
||||
|
||||
test "iter seek":
|
||||
initData(db)
|
||||
check(toSeq(db.iter(seek = "ab")) ==
|
||||
@[("ba", "2"), ("bb", "3")])
|
||||
|
||||
test "iter seek reverse":
|
||||
initData(db)
|
||||
check(toSeq(db.iter(seek = "ab", reverse = true)) ==
|
||||
@[("ba", "2"), ("aa", "1")])
|
||||
|
||||
test "iter prefix":
|
||||
initData(db)
|
||||
check(toSeq(db.iterPrefix(prefix = "b")) ==
|
||||
@[("ba", "2"), ("bb", "3")])
|
||||
|
||||
test "iter range":
|
||||
initData(db)
|
||||
check(toSeq(db.iterRange(start = "a", limit = "ba")) ==
|
||||
@[("aa", "1"), ("ba", "2")])
|
||||
|
||||
test "iter range reverse":
|
||||
initData(db)
|
||||
check(toSeq(db.iterRange(start = "bb", limit = "b")) ==
|
||||
@[("bb", "3"), ("ba", "2")])
|
||||
|
||||
test "iter with 0x00":
|
||||
db.put("\0z1", "\0ff")
|
||||
db.put("z2\0", "ff\0")
|
||||
check(toSeq(db.iter()) == @[("\0z1", "\0ff"), ("z2\0", "ff\0")])
|
||||
|
||||
test "iter empty value":
|
||||
db.put("a", "")
|
||||
check(toSeq(db.iter()) == @[("a", "")])
|
||||
|
||||
test "iter empty key":
|
||||
db.put("", "a")
|
||||
check(toSeq(db.iter()) == @[("", "a")])
|
||||
|
||||
test "repair database":
|
||||
initData(db)
|
||||
db.close()
|
||||
repairDb(dbName)
|
||||
|
||||
test "batch":
|
||||
db.put("a", "1")
|
||||
db.put("b", "2")
|
||||
let batch = newBatch()
|
||||
batch.put("a", "10")
|
||||
batch.put("c", "30")
|
||||
batch.delete("b")
|
||||
db.write(batch)
|
||||
check(toSeq(db.iter()) == @[("a", "10"), ("c", "30")])
|
||||
|
||||
test "batch append":
|
||||
let batch = newBatch()
|
||||
let batch2 = newBatch()
|
||||
batch.put("a", "1")
|
||||
batch2.put("b", "2")
|
||||
batch2.delete("a")
|
||||
batch.append(batch2)
|
||||
db.write(batch)
|
||||
check(toSeq(db.iter()) == @[("b", "2")])
|
||||
|
||||
test "batch clear":
|
||||
let batch = newBatch()
|
||||
batch.put("a", "1")
|
||||
batch.clear()
|
||||
batch.put("b", "2")
|
||||
db.write(batch)
|
||||
check(toSeq(db.iter()) == @[("b", "2")])
|
||||
|
||||
test "open with cache":
|
||||
let ldb = leveldb.open(dbName & "-cache", cacheCapacity = 100000)
|
||||
defer:
|
||||
ldb.close()
|
||||
removeDb(ldb.path)
|
||||
ldb.put("a", "1")
|
||||
check(toSeq(ldb.iter()) == @[("a", "1")])
|
||||
|
||||
test "open but no create":
|
||||
expect LevelDbException:
|
||||
let failed = leveldb.open(dbName & "-nocreate", create = false)
|
||||
defer:
|
||||
failed.close()
|
||||
removeDb(failed.path)
|
||||
|
||||
test "open but no reuse":
|
||||
let old = leveldb.open(dbName & "-noreuse", reuse = true)
|
||||
defer:
|
||||
old.close()
|
||||
removeDb(old.path)
|
||||
|
||||
expect LevelDbException:
|
||||
let failed = leveldb.open(old.path, reuse = false)
|
||||
defer:
|
||||
failed.close()
|
||||
removeDb(failed.path)
|
||||
|
||||
test "no compress":
|
||||
db.close()
|
||||
let nc = leveldb.open(dbName, compressionType = ctNoCompression)
|
||||
defer: nc.close()
|
||||
nc.put("a", "1")
|
||||
check(toSeq(nc.iter()) == @[("a", "1")])
|
||||
|
||||
suite "package":
|
||||
|
||||
setup:
|
||||
removeDir(tmpDir)
|
||||
|
||||
test "import as package":
|
||||
let (output, exitCode) = execNimble("install")
|
||||
check exitCode == QuitSuccess
|
||||
check output.contains("leveldb installed successfully.")
|
||||
|
||||
cd "tests/packagetest":
|
||||
var (output, exitCode) = execNimble("build")
|
||||
check exitCode == QuitSuccess
|
||||
check output.contains("Building")
|
||||
|
||||
(output, exitCode) = execCmdEx("./packagetest")
|
||||
checkpoint output
|
||||
check exitCode == QuitSuccess
|
||||
check output.contains("leveldb works.")
|
||||
|
||||
suite "tool":
|
||||
|
||||
setup:
|
||||
removeDir(tmpDir)
|
||||
|
||||
test "leveldb tool":
|
||||
var (output, exitCode) = execNimble("install")
|
||||
check exitCode == QuitSuccess
|
||||
check output.contains("Building")
|
||||
|
||||
check execTool("-v").exitCode == QuitSuccess
|
||||
check execTool("create").exitCode == QuitSuccess
|
||||
check execTool("list").exitCode == QuitSuccess
|
||||
|
||||
check execTool("put", "hello", "world").exitCode == QuitSuccess
|
||||
(output, exitCode) = execTool("get", "hello")
|
||||
check exitCode == QuitSuccess
|
||||
check output == "world\L"
|
||||
(output, exitCode) = execTool("list")
|
||||
check exitCode == QuitSuccess
|
||||
check output == "hello world\L"
|
||||
|
||||
check execTool("delete", "hello").exitCode == QuitSuccess
|
||||
(output, exitCode) = execTool("get", "hello")
|
||||
check exitCode == QuitSuccess
|
||||
check output == ""
|
||||
(output, exitCode) = execTool("list")
|
||||
check exitCode == QuitSuccess
|
||||
check output == ""
|
||||
|
||||
check execTool("put", "hello", "6130", "-x").exitCode == QuitSuccess
|
||||
check execTool("get", "hello", "-x").output == "6130\L"
|
||||
check execTool("get", "hello").output == "a0\L"
|
||||
check execTool("list", "-x").output == "hello 6130\L"
|
||||
check execTool("put", "hello", "0061", "-x").exitCode == QuitSuccess
|
||||
check execTool("get", "hello", "-x").output == "0061\L"
|
||||
check execTool("delete", "hello").exitCode == QuitSuccess
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
import std/monotimes
|
||||
import os
|
||||
|
||||
import pkg/chronos
|
||||
import pkg/questionable
|
||||
import pkg/questionable/results
|
||||
import pkg/libp2p/cid
|
||||
import pkg/datastore
|
||||
|
||||
import pkg/codex/rng
|
||||
import pkg/codex/chunker
|
||||
import pkg/codex/blocktype as bt
|
||||
|
||||
import ../leveldb/leveldbds
|
||||
|
||||
import ./asynctest
|
||||
import ./checktest
|
||||
import ./helpers
|
||||
import ./codex/helpers
|
||||
|
||||
proc setGetTest(store: DataStore) {.async.} =
|
||||
let chunker = RandomChunker.new(Rng.instance(), size = 4096000, chunkSize = 256)
|
||||
var blocks: seq[bt.Block]
|
||||
while true:
|
||||
let chunk = await chunker.getBytes()
|
||||
if chunk.len <= 0:
|
||||
break
|
||||
blocks.add(bt.Block.new(chunk).tryGet())
|
||||
|
||||
for blk in blocks:
|
||||
let key = Key.init($blk.cid).tryGet()
|
||||
discard (await store.put(key, blk.data))
|
||||
let bytes = (await store.get(key)).tryGet()
|
||||
check:
|
||||
bytes == blk.data
|
||||
|
||||
proc doTest(name: string, store: DataStore) {.async.} =
|
||||
let chunker = RandomChunker.new(Rng.instance(), size = 4096000, chunkSize = 256)
|
||||
var blocks: seq[bt.Block]
|
||||
while true:
|
||||
let chunk = await chunker.getBytes()
|
||||
if chunk.len <= 0:
|
||||
break
|
||||
blocks.add(bt.Block.new(chunk).tryGet())
|
||||
|
||||
let t0 = getMonoTime()
|
||||
for blk in blocks:
|
||||
let key = Key.init($blk.cid).tryGet()
|
||||
discard (await store.put(key, blk.data))
|
||||
|
||||
let t1 = getMonoTime()
|
||||
|
||||
var read: seq[seq[byte]]
|
||||
for blk in blocks:
|
||||
let key = Key.init($blk.cid).tryGet()
|
||||
let bytes = (await store.get(key)).tryGet()
|
||||
read.add(bytes)
|
||||
|
||||
let t2 = getMonoTime()
|
||||
|
||||
for i in 0..<blocks.len:
|
||||
check:
|
||||
blocks[i].data == read[i]
|
||||
|
||||
setGetTest(store)
|
||||
|
||||
echo name & " = " & $(t1 - t0) & " / " & $(t2 - t1)
|
||||
|
||||
asyncchecksuite "SQL":
|
||||
test "should A":
|
||||
await doTest("defaultSQL", SQLiteDatastore.new("defaultSQL").tryGet())
|
||||
|
||||
let dir = "defaultFS"
|
||||
if not dirExists(dir):
|
||||
createDir(dir)
|
||||
await doTest("defaultFS", FSDatastore.new(dir, depth = 5).tryGet())
|
||||
removeDir(dir)
|
||||
|
||||
let ldb = LevelDbDatastore.new("leveldb").tryGet()
|
||||
await doTest("leveldb", ldb)
|
||||
|
||||
Loading…
Reference in New Issue