Redis+KVStore: Disk-based Storage for Massive Data

Encoding Mode

[<namespace>] <key> <type> <element…>

Value Encoding Format

<type> <element…>

Encoding of various data types

KeyObject ValueObject

String [<ns>] <key> KEY_META KEY_STRING <MetaObject>

Hash [<ns>] <key> KEY_META KEY_HASH <MetaObject>

[<ns>] <key> KEY_HASH_FIELD <field> KEY_HASH_FIELD <field-value>

Set [<ns>] <key> KEY_META KEY_SET <MetaObject>

[<ns>] <key> KEY_SET_MEMBER <member> KEY_SET_MEMBER

List [<ns>] <key> KEY_META KEY_LIST <MetaObject>

[<ns>] <key> KEY_LIST_ELEMENT <index> KEY_LIST_ELEMENT <element-value>

Sorted Set [<ns>] <key> KEY_META KEY_ZSET <MetaObject>

[<ns>] <key> KEY_ZSET_SCORE <member> KEY_ZSET_SCORE <score>

[<ns>] <key> KEY_ZSET_SORT <score> <member> KEY_ZSET_SORT

ZSet encoding example

// // The “|” in the pseudo-code represents the division of the domain and does not mean to store the data as “|”. During actual serialization, each field is serialized at a specific location.

The key is: ns|1|A (1 stands for KEY_META metadata type).

The value is: metadata encoding (redis data type/zset, expiration time, number of members, maximum and minimum score among others).

The key is: ns|11|A|first (11 stands for the KEY_ZSET_SCORE type).

The value is: 11|1 (11 stands for the KEY_ZSET_SCORE type. 1 stands for the score of the Member “first”).

The key is: ns|10|A|1|first (10 stands for the KEY_ZSET_SORT type and 1 stands for the score).

The value is: 10 (representing the KEY_ZSET_SORT type, insignificant. RocksDB automatically sorts the values by key, so it is easy to calculate the rank, requiring no storage and updating).

KeyObject meta_key(ctx.ns, KEY_META, key);

ValueObject meta_value;

for (each_member) {

// KEY_ZSET_SORT stores the rank information.

KeyObject zsort(ctx.ns, KEY_ZSET_SORT, key);

zsort.SetZSetMember(str);

zsort.SetZSetScore(score);

ValueObject zsort_value;

zsort_value.SetType(KEY_ZSET_SORT);

GetDBWriter().Put(ctx, zsort, zsort_value);

// Store the score information.

KeyObject zscore(ctx.ns, KEY_ZSET_SCORE, key);

zscore.SetZSetMember(str);

ValueObject zscore_value;

zscore_value.SetType(KEY_ZSET_SCORE);

zscore_value.SetZSetScore(score);

GetDBWriter().Put(ctx, zscore, zscore_value);

}

if (expiretime > 0)

{

meta_value.SetTTL(expiretime);

}

// Metadata.

GetDBWriter().Put(ctx, meta_key, meta_value);

Implementation of Del

int Ardb::DelKey(Context& ctx, const KeyObject& meta_key, Iterator*& iter)

{

ValueObject meta_obj003B

if (0 == m_engine->Get(ctx, meta_key, meta_obj))

{

// Delete directly if the data is of the string type.

if (meta_obj.GetType() == KEY_STRING)

{

int err = RemoveKey(ctx, meta_key);

return err == 0 ? 1 : 0;

}

}

else

{

return 0;

}

if (NULL == iter)

{

// If the data is of a complicated type, the database will be traversed based on the namespace, key and type prefix.

// Search all the members with the prefix of namespace|type|Key.

iter = m_engine->Find(ctx, meta_key);

}

else

{

iter->Jump(meta_key);

}

while (NULL != iter && iter->Valid())

{

KeyObject& k = iter->Key();

iter->Del();

iter->Next();

}

}

Iterator* RocksDBEngine::Find(Context& ctx, const KeyObject& key) {

opt.prefix_same_as_start = true;

if (!ctx.flags.iterate_no_upperbound)

{

KeyObject& upperbound_key = iter->IterateUpperBoundKey();

upperbound_key.SetNameSpace(key.GetNameSpace());

if (key.GetType() == KEY_META)

{

upperbound_key.SetType(KEY_END);

}

else

{

upperbound_key.SetType(key.GetType() + 1);

}

upperbound_key.SetKey(key.GetKey());

upperbound_key.CloneStringPart();

}

}

Implementation of Expire

Concluding remarks

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store