Verify: migrate from Amazon ElastiCache to KiviDB

Compare key counts first. On a cluster-mode source, run INFO keyspace on each shard's primary and add them up.

redis-cli -h master.my-cache.xxxxxx.use1.cache.amazonaws.com --tls --user app --pass "$PASSWORD" INFO keyspace
redis-cli -h my-db.cloud.kividb.io -p 6380 --tls --cacert kividb-ca.pem --user kividbapp --pass "$KIVIDB_PASSWORD" INFO keyspace

Then spot-check random keys: their type, their value and their TTL. This script does it, and works against a cluster on either side. It needs pip install redis.

"""usage: python3 verify.py SOURCE_URL TARGET_URL [N]
SOURCE_CLUSTER=1 / TARGET_CLUSTER=1: that side is a cluster.
SOURCE_CA / TARGET_CA: a CA file for a rediss:// URL."""
import os, sys
import redis
from redis.cluster import RedisCluster


def connect(url, side):
    kw = {}
    if url.startswith("rediss://") and os.environ.get(f"{side}_CA"):
        kw["ssl_ca_certs"] = os.environ[f"{side}_CA"]
    if os.environ.get(f"{side}_CLUSTER") == "1":
        return RedisCluster.from_url(url, **kw)
    return redis.Redis.from_url(url, **kw)


def dbsize(r):
    if isinstance(r, RedisCluster):
        n = r.dbsize(target_nodes=RedisCluster.PRIMARIES)
    else:
        n = r.dbsize()
    return sum(n.values()) if isinstance(n, dict) else n


def value(r, key, t):
    if t == "string":
        v = r.get(key)
        return ("hll", r.pfcount(key)) if v and v.startswith(b"HYLL") else v
    if t == "hash":
        return r.hgetall(key)
    if t == "list":
        return r.lrange(key, 0, -1)
    if t == "set":
        return sorted(r.smembers(key))
    if t == "zset":
        return r.zrange(key, 0, -1, withscores=True)
    if t == "stream":
        return r.xrange(key)
    if t == "ReJSON-RL":
        return r.execute_command("JSON.GET", key)
    return None  # other types: compared by type only


src = connect(sys.argv[1], "SOURCE")
dst = connect(sys.argv[2], "TARGET")
n = int(sys.argv[3]) if len(sys.argv) > 3 else 1000

s_size, d_size = dbsize(src), dbsize(dst)
print(f"keys: source {s_size}, target {d_size}")

bad = 0
for _ in range(n):
    key = src.randomkey()
    if key is None:
        break
    t = src.type(key).decode()
    t2 = dst.type(key).decode()
    if t != t2:
        bad += 1
        print(f"TYPE  {key!r}: source {t}, target {t2}")
        continue
    if value(src, key, t) != value(dst, key, t):
        bad += 1
        print(f"VALUE {key!r} ({t}) differs")
    s_ttl, d_ttl = src.pttl(key), dst.pttl(key)
    if (s_ttl < 0) != (d_ttl < 0) or abs(s_ttl - d_ttl) > 60_000:
        bad += 1
        print(f"TTL   {key!r}: source {s_ttl} ms, target {d_ttl} ms")

print(f"checked {n} random keys, {bad} mismatches")
sys.exit(1 if bad or s_size != d_size else 0)

SOURCE_CLUSTER=1 TARGET_CA=kividb-ca.pem python3 verify.py \
  "rediss://app:$PASSWORD@clustercfg.my-cluster.xxxxxx.use1.cache.amazonaws.com:6379" \
  "rediss://kividbapp:$KIVIDB_PASSWORD@my-db.cloud.kividb.io:6380" 2000

HyperLogLogs are compared by their count, not their bytes. Run it from the same EC2 instance as the copy. Against a source that is still taking writes, expect a few differences on keys that changed in between; with writes stopped, expect none.