Feature #551
LMDB with shm_open instead of disk file
Status: | New | Start date: | 05/13/2020 | |
---|---|---|---|---|
Priority: | Normal (Code 4) | Due date: | ||
Assignee: | - | % Done: | 0% | |
Category: | - | |||
Target version: | - |
Description
For environments such as PCI/DSS there shall be no data stored on disk. Thus even with MDB_NOSYNC and MDB_NOMETASYNC there is no guarantee that data does not pass to disk.
LMDB on tmpfs might cause results like here: https://github.com/deephacks/lmdbjni/issues/89
Thus we could patch the lib to use shm_open for data part. For locks still the classical open() hdd based files shall be used to allow fcntl(env->me_lfd, F_SETLK to work.
Also when using shm_open, the full map size shall be intialized by ftruncate(). Done by MDB_WRITEMAP or needs some changes in code.
The quick and dirty patch looks like:
$ git diff diff --git a/libatmi/atmi_cache_init.c b/libatmi/atmi_cache_init.c index a1aa26c..fc4895c 100644 --- a/libatmi/atmi_cache_init.c +++ b/libatmi/atmi_cache_init.c @@ -348,8 +348,8 @@ exprivate int sort_data_bydate(const EDB_val *a, const EDB_val *b) exprivate int ndrx_cache_phydb_getref(ndrx_tpcache_db_t *db) { int ret = EXSUCCEED; - unsigned int dbflags=0; - ndrx_tpcache_phydb_t *phy; + unsigned int dbflags=EDB_WRITEMAP; + ndrx_tpcache_phydb_t *phy=NULL; if (NULL!=(db->phy = ndrx_cache_phydbget(db->cachedbphy))) { @@ -435,6 +435,12 @@ exprivate int ndrx_cache_phydb_getref(ndrx_tpcache_db_t *db) db->phy = phy; out: + /* avoid memory leak */ + if (NULL!=phy && EXSUCCEED!=ret) + { + NDRX_FREE(phy); + } + return ret; } diff --git a/libnstd/lmdb/edb.c b/libnstd/lmdb/edb.c index 15a1824..126ca92 100644 --- a/libnstd/lmdb/edb.c +++ b/libnstd/lmdb/edb.c @@ -568,7 +568,7 @@ typedef EDB_ID txnid_t; * Set this to 1 for copious tracing. Set to 2 to add dumps of all IDLs * read from and written to the database (used for free space management). */ -#define EDB_DEBUG 0 +#define EDB_DEBUG 1 #endif #if EDB_DEBUG @@ -3718,6 +3718,8 @@ edb_page_flush(EDB_txn *txn, int keep) return rc; } #else + +#if 0 /* Write up to EDB_COMMIT_PAGES dirty pages at a time. */ if (pos!=next_pos || n==EDB_COMMIT_PAGES || wsize+size>MAX_WRITE) { if (n) { @@ -3759,6 +3761,7 @@ retry_seek: wpos = pos; wsize = 0; } +#endif DPRINTF(("committing page %"Yu, pgno)); next_pos = pos + size; iov[n].iov_len = size; @@ -4647,6 +4650,7 @@ edb_fopen(const EDB_env *env, EDB_name *fname, { int rc = EDB_SUCCESS; HANDLE fd; + char *nm; #ifdef _WIN32 DWORD acc, share, disp, attrs; #else @@ -4696,11 +4700,26 @@ edb_fopen(const EDB_env *env, EDB_name *fname, } fd = CreateFileW(fname->mn_val, acc, share, NULL, disp, attrs, NULL); #else - fd = open(fname->mn_val, which & EDB_O_MASK, mode); + /* only file name ... */ + nm = strrchr(fname->mn_val, '/')+1; + + if (0==strcmp(nm, "lock.edb")) + { + fd = open(fname->mn_val, which & EDB_O_MASK, mode); + } + else + { + fd = shm_open(nm, which & EDB_O_MASK, mode); + } #endif if (fd == INVALID_HANDLE_VALUE) rc = ErrCode(); + /* only for locks file... */ + else if (0!=strcmp(nm, "lock.edb")) + { + /* do nothing... */ + } #ifndef _WIN32 else { if (which != EDB_O_RDONLY && which != EDB_O_RDWR) {
History
#1 Updated by Madars almost 4 years ago
- File patch.out added