Semantic search inside the server · Redis 8.10

1  $ redis-cli INFO server | head -20; redis-cli MODULE LIST; head -c 400 /data/articles.jsonl; wc -l /data/articles.jsonl; python3 -c "import redis; print(redis.__version__)"; grep -v '^#' /etc/redis/redis.conf | grep -v '^$' | head -50

2  $ redis-cli DBSIZE; redis-cli FT._LIST; redis-cli CONFIG GET save; redis-cli CONFIG GET appendonly; redis-cli CONFIG GET maxmemory*; python3 -c "
import json,collections,math
rows=[json.loads(l) for l in open('/data/articles.jsonl') if l.strip()]
print(len(rows), collections.Counter(len(r['embedding']) for r in rows))
ids=[r['id'] for r in rows]; print(len(set(ids)))
print(collections.Counter(type(x).__name__ for r in rows for x in r['embedding']))
print(sum(1 for r in rows if all(x==0 for x in r['embedding'])))
print(min(math.sqrt(sum(x*x for x in r['embedding'])) for r in rows), max(math.sqrt(sum(x*x for x in r['embedding'])) for r in rows))
print(set(r.keys()) for r in rows[:1])
"

3  $ cat > /tmp/load.py <<'EOF'
import json, struct, redis
r = redis.Redis(host='127.0.0.1', port=6379)
try: r.execute_command('FT.DROPINDEX', 'idx:articles', 'DD')
except redis.ResponseError: pass
r.execute_command('FT.CREATE','idx:articles','ON','HASH','PREFIX','1','article:',
  'SCHEMA','id','TAG','title','TEXT','embedding','VECTOR','FLAT','6','TYPE','FLOAT32','DIM','64','DISTANCE_METRIC','COSINE')
p = r.pipeline(transaction=False)
n=0
for line in open('/data/articles.jsonl'):
    if not line.strip(): continue
    a = json.loads(line)
    p.hset('article:'+a['id'], mapping={'id':a['id'],'title':a['title'],
        'embedding':struct.pack('<64f',*a['embedding'])}); n+=1
p.execute(); print(n)
EOF
python3 /tmp/load.py
mkdir -p /app && cat > /app/search.py <<'EOF'
import json, struct, sys, redis
q = json.load(sys.stdin)
if len(q) != 64: sys.exit("query must have 64 floats")
r = redis.Redis(host='127.0.0.1', port=6379, decode_responses=False)
res = r.execute_command('FT.SEARCH', 'idx:articles', '*=>[KNN 5 @embedding $v AS score]',
    'PARAMS', '2', 'v', struct.pack('<64f', *q),
    'SORTBY', 'score', 'ASC', 'RETURN', '2', 'id', 'score', 'LIMIT', '0', '5', 'DIALECT', '2')
for i in range(1, len(res), 2):
    f = dict(zip(res[i+1][::2], res[i+1][1::2]))
    print(f[b'id'].decode())
EOF
sleep 1; redis-cli FT.INFO idx:articles | grep -A1 -E 'num_docs|hash_indexing_failures|percent_indexed'

4  $ cd /tmp && python3 -c "
import json,random,subprocess,math
rows=[json.loads(l) for l in open('/data/articles.jsonl')]
random.seed(1); bad=0
qs=[r['embedding'] for r in random.sample(rows,10)]+[[random.gauss(0,1) for _ in range(64)] for _ in range(20)]
for q in qs:
    n=math.sqrt(sum(x*x for x in q))
    sims=sorted(((sum(a*b for a,b in zip(q,r['embedding']))/n/math.sqrt(sum(b*b for b in r['embedding'])),r['id']) for r in rows),reverse=True)
    exp=[i for _,i in sims[:5]]
    got=subprocess.run(['python3','/app/search.py'],input=json.dumps(q),capture_output=True,text=True).stdout.split()
    if got!=exp: bad+=1; print('MISMATCH',got,exp,[round(s,6) for s,_ in sims[:6]])
print('mismatches',bad,'of',len(qs))
"

5  $ cd /tmp && python3 -c "import json;print(json.dumps(json.loads(open('/data/articles.jsonl').readline())['embedding']))" > q.json && python3 /app/search.py < q.json

6  $ cd /tmp && python3 -c "
import redis,struct,json
r=redis.Redis(); q=json.load(open('q.json'))
print(r.connection_pool.connection_kwargs.get('protocol'))
print(r.execute_command('FT.SEARCH','idx:articles','*=>[KNN 5 @embedding \$v AS score]','PARAMS','2','v',struct.pack('<64f',*q),'SORTBY','score','ASC','RETURN','2','id','score','DIALECT','2'))"

7  $ sed -i "s/decode_responses=False)/decode_responses=False, protocol=2)/" /app/search.py && cat /app/search.py && python3 /app/search.py < /tmp/q.json

8  $ cd /tmp && python3 -c "
import json,random,subprocess,math
rows=[json.loads(l) for l in open('/data/articles.jsonl')]
random.seed(2); bad=0
qs=[r['embedding'] for r in random.sample(rows,20)]+[[random.gauss(0,1) for _ in range(64)] for _ in range(40)]
for q in qs:
    n=math.sqrt(sum(x*x for x in q))
    sims=sorted(((sum(a*b for a,b in zip(q,r['embedding']))/n/math.sqrt(sum(b*b for b in r['embedding'])),r['id']) for r in rows),reverse=True)
    exp=[i for _,i in sims[:5]]
    got=subprocess.run(['python3','/app/search.py'],input=json.dumps(q),capture_output=True,text=True).stdout.split()
    if got!=exp: bad+=1; print('MISMATCH',got,exp,[round(s,7) for s,_ in sims[:6]])
print('mismatches',bad,'of',len(qs))
"

9  $ redis-cli SAVE; ls -la /var/lib/redis; redis-cli DBSIZE

{
 "dbsize": 1000,
 "queries": [
  {
   "exit": 0,
   "ids": [
    "kb-0468",
    "kb-0368",
    "kb-0028",
    "kb-0088",
    "kb-0708"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0215",
    "kb-0155",
    "kb-0315",
    "kb-0535",
    "kb-0615"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0963",
    "kb-0123",
    "kb-0423",
    "kb-0583",
    "kb-0823"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0237",
    "kb-0597",
    "kb-0097",
    "kb-0337",
    "kb-0837"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0541",
    "kb-0621",
    "kb-0841",
    "kb-0581",
    "kb-0481"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  }
 ],
 "server": {
  "redis_version": "8.10.2",
  "server_name": null,
  "valkey_version": null
 },
 "server_side_search_commands": [
  "FT.SEARCH"
 ]
}

1  $ redis-cli INFO server | head -5; redis-cli MODULE LIST; head -c 300 /data/articles.jsonl; echo; wc -l /data/articles.jsonl; python3 -c "import redis; print(redis.__version__)"; python3 -c "import numpy; print(numpy.__version__)"; grep -vE '^\s*(#|$)' /etc/redis/redis.conf

2  $ redis-cli CONFIG GET save; redis-cli CONFIG GET appendonly; redis-cli CONFIG GET maxmemory*; redis-cli DBSIZE; redis-cli FT._LIST; redis-cli --scan | head; python3 - <<'E'
import json,collections,math
ids=collections.Counter();bad=0;zero=0
for l in open('/data/articles.jsonl'):
    o=json.loads(l);ids[o['id']]+=1
    e=o['embedding']
    if len(e)!=64 or not all(isinstance(x,(int,float)) and math.isfinite(x) for x in e): bad+=1;print(o['id'],len(e))
    elif sum(x*x for x in e)==0: zero+=1
print(len(ids),[k for k,v in ids.items() if v>1],bad,zero)
E

3  $ python3 - <<'E'
import json,redis,numpy as np
r=redis.Redis()
r.execute_command('FT.CREATE','idx:articles','ON','HASH','PREFIX','1','article:','SCHEMA','id','TAG','title','TEXT','embedding','VECTOR','FLAT','6','TYPE','FLOAT32','DIM','64','DISTANCE_METRIC','COSINE')
p=r.pipeline(transaction=False)
for l in open('/data/articles.jsonl'):
    o=json.loads(l)
    p.hset('article:'+o['id'],mapping={'id':o['id'],'title':o['title'],'embedding':np.array(o['embedding'],dtype=np.float32).tobytes()})
p.execute()
E
sleep 1; redis-cli FT.INFO idx:articles | grep -A1 -E 'num_docs|hash_indexing_failures|percent_indexed'
mkdir -p /app; cat > /app/search.py <<'E'
import json, sys
import numpy as np
import redis

q = np.array(json.load(sys.stdin), dtype=np.float32)
if q.shape != (64,):
    sys.exit("query must be a list of 64 floats")
r = redis.Redis(host="127.0.0.1", port=6379)
res = r.execute_command(
    "FT.SEARCH", "idx:articles", "*=>[KNN 5 @embedding $vec AS score]",
    "PARAMS", "2", "vec", q.tobytes(),
    "SORTBY", "score", "ASC", "RETURN", "1", "id", "LIMIT", "0", "5", "DIALECT", "2",
)
for i in range(2, len(res), 2):
    f = res[i]
    d = {f[j]: f[j + 1] for j in range(0, len(f), 2)}
    print(d[b"id"].decode())
E
redis-cli SAVE

4  $ cd /tmp && python3 - <<'E'
import json,subprocess,numpy as np
A=[json.loads(l) for l in open('/data/articles.jsonl')]
ids=[a['id'] for a in A];M=np.array([a['embedding'] for a in A]);Mn=M/np.linalg.norm(M,axis=1,keepdims=True)
rng=np.random.default_rng(0);fail=0
qs=[rng.normal(size=64) for _ in range(20)]+[M[i]+rng.normal(scale=.05,size=64) for i in range(0,1000,50)]
for q in qs:
    out=subprocess.run(['python3','/app/search.py'],input=json.dumps(q.tolist()),capture_output=True,text=True).stdout.split()
    s=Mn@(q/np.linalg.norm(q));exp=[ids[i] for i in np.argsort(-s)[:5]]
    if out!=exp: fail+=1;print(out,exp,sorted(s)[-6:])
print("fail",fail,"of",len(qs))
E

5  $ cd /tmp && python3 -c "import json;print(json.dumps([0.1]*64))" > q.json && python3 /app/search.py < q.json

6  $ cd /tmp && python3 -c "
import redis,numpy as np
r=redis.Redis()
print(r.connection_pool.connection_kwargs)
res=r.execute_command('FT.SEARCH','idx:articles','*=>[KNN 5 @embedding \$vec AS score]','PARAMS','2','vec',np.full(64,.1,dtype=np.float32).tobytes(),'SORTBY','score','ASC','RETURN','2','id','score','LIMIT','0','5','DIALECT','2')
print(type(res),res)"

7  $ sed -i 's/r = redis.Redis(host="127.0.0.1", port=6379)/r = redis.Redis(host="127.0.0.1", port=6379, protocol=2)/' /app/search.py && cd /tmp && python3 /app/search.py < q.json && python3 - <<'E'
import json,subprocess,numpy as np
A=[json.loads(l) for l in open('/data/articles.jsonl')]
ids=[a['id'] for a in A];M=np.array([a['embedding'] for a in A]);Mn=M/np.linalg.norm(M,axis=1,keepdims=True)
rng=np.random.default_rng(0);fail=0
qs=[rng.normal(size=64) for _ in range(30)]+[M[i]+rng.normal(scale=.05,size=64) for i in range(0,1000,50)]+[M[7]*3]
for q in qs:
    out=subprocess.run(['python3','/app/search.py'],input=json.dumps(q.tolist()),capture_output=True,text=True).stdout.split()
    s=Mn@(q/np.linalg.norm(q));o=np.argsort(-s);exp=[ids[i] for i in o[:5]]
    if out!=exp: fail+=1;print(out,exp,s[o[:6]])
print("fail",fail,"of",len(qs))
E

8  $ redis-cli SAVE; redis-cli DBSIZE; cat /app/search.py

{
 "dbsize": 1000,
 "queries": [
  {
   "exit": 0,
   "ids": [
    "kb-0468",
    "kb-0368",
    "kb-0028",
    "kb-0088",
    "kb-0708"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0215",
    "kb-0155",
    "kb-0315",
    "kb-0535",
    "kb-0615"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0963",
    "kb-0123",
    "kb-0423",
    "kb-0583",
    "kb-0823"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0237",
    "kb-0597",
    "kb-0097",
    "kb-0337",
    "kb-0837"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0541",
    "kb-0621",
    "kb-0841",
    "kb-0581",
    "kb-0481"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  }
 ],
 "server": {
  "redis_version": "8.10.2",
  "server_name": null,
  "valkey_version": null
 },
 "server_side_search_commands": [
  "FT.SEARCH"
 ]
}

1  $ redis-cli INFO server | head -20; redis-cli MODULE LIST; head -c 300 /data/articles.jsonl; echo; wc -l /data/articles.jsonl; python3 -c "import redis; print(redis.__version__)"; grep -v '^#' /etc/redis/redis.conf | grep -v '^$' | head -50

2  $ redis-cli CONFIG GET save; redis-cli CONFIG GET appendonly; redis-cli DBSIZE; redis-cli FT._LIST; redis-cli CONFIG GET maxmemory*; python3 -c "
import json,collections,math
ids=collections.Counter();bad=0
for l in open('/data/articles.jsonl'):
    d=json.loads(l);ids[d['id']]+=1
    e=d.get('embedding')
    if not isinstance(e,list) or len(e)!=64 or any(not isinstance(x,(int,float)) or math.isnan(x) for x in e) or all(x==0 for x in e): bad+=1;print(d.get('id'),type(e), len(e) if isinstance(e,list) else None)
print('dups',[k for k,v in ids.items() if v>1],'bad',bad, set(json.loads(open('/data/articles.jsonl').readline()).keys()))
"

3  $ cat > /app/load.py <<'EOF'
import json, struct, redis
r = redis.Redis(host='127.0.0.1', port=6379)
p = r.pipeline(transaction=False)
for line in open('/data/articles.jsonl'):
    d = json.loads(line)
    p.hset(f"article:{d['id']}", mapping={'id': d['id'], 'title': d['title'],
           'embedding': struct.pack('<64f', *d['embedding'])})
p.execute()
try: r.execute_command('FT.DROPINDEX', 'idx:articles')
except redis.ResponseError: pass
r.execute_command('FT.CREATE','idx:articles','ON','HASH','PREFIX','1','article:',
  'SCHEMA','id','TAG','title','TEXT','embedding','VECTOR','FLAT','6',
  'TYPE','FLOAT32','DIM','64','DISTANCE_METRIC','COSINE')
EOF
cat > /app/search.py <<'EOF'
import sys, json, struct, redis
q = json.load(sys.stdin)
r = redis.Redis(host='127.0.0.1', port=6379, decode_responses=False)
res = r.execute_command('FT.SEARCH', 'idx:articles', '*=>[KNN 5 @embedding $v AS score]',
    'PARAMS', '2', 'v', struct.pack(f'<{len(q)}f', *q),
    'SORTBY', 'score', 'ASC', 'RETURN', '2', 'id', 'score', 'LIMIT', '0', '5', 'DIALECT', '2')
for i in range(2, len(res), 2):
    f = dict(zip(res[i][::2], res[i][1::2]))
    print(f[b'id'].decode())
EOF
python3 /app/load.py && sleep 1 && redis-cli FT.INFO idx:articles | grep -A1 -E 'num_docs|hash_indexing_failures|percent_indexed'

4  $ cd /tmp && python3 -c "
import json,random,subprocess,math
A=[json.loads(l) for l in open('/data/articles.jsonl')]
def cos(a,b): return sum(x*y for x,y in zip(a,b))/math.sqrt(sum(x*x for x in a)*sum(y*y for y in b))
random.seed(1); bad=0
for t in range(40):
    q=random.choice(A)['embedding'] if t<20 else [random.gauss(0,1) for _ in range(64)]
    if t<20: q=[x+random.gauss(0,0.02) for x in q]
    exp=[a['id'] for a in sorted(A,key=lambda a:-cos(q,a['embedding']))[:5]]
    got=subprocess.run(['python3','/app/search.py'],input=json.dumps(q),capture_output=True,text=True).stdout.split()
    if got!=exp:
        bad+=1; print(t,exp,got,[round(cos(q,a['embedding']),6) for a in sorted(A,key=lambda a:-cos(q,a['embedding']))[:6]])
print('mismatches',bad)
"

5  $ cd /tmp && python3 -c "
import json;print(json.dumps(json.loads(open('/data/articles.jsonl').readline())['embedding']))" > q.json; python3 /app/search.py < q.json

6  $ cd /tmp && python3 -c "
import json,struct,redis
q=json.load(open('q.json'))
r=redis.Redis()
print(r.connection_pool.connection_kwargs)
print(r.execute_command('FT.SEARCH','idx:articles','*=>[KNN 5 @embedding \$v AS score]','PARAMS','2','v',struct.pack('<64f',*q),'SORTBY','score','ASC','RETURN','2','id','score','LIMIT','0','5','DIALECT','2'))"

7  $ cat > /app/search.py <<'EOF'
import sys, json, struct, redis
q = json.load(sys.stdin)
r = redis.Redis(host='127.0.0.1', port=6379, protocol=2)
res = r.execute_command('FT.SEARCH', 'idx:articles', '*=>[KNN 5 @embedding $v AS score]',
    'PARAMS', '2', 'v', struct.pack(f'<{len(q)}f', *q),
    'SORTBY', 'score', 'ASC', 'RETURN', '2', 'id', 'score', 'LIMIT', '0', '5', 'DIALECT', '2')
if isinstance(res, dict):  # RESP3 reply shape
    rows = [x[b'extra_attributes'] for x in res[b'results']]
else:                      # RESP2 reply shape
    rows = [dict(zip(res[i][::2], res[i][1::2])) for i in range(2, len(res), 2)]
for f in rows:
    print(f[b'id'].decode())
EOF
cd /tmp && python3 /app/search.py < q.json && python3 -c "
import json,random,subprocess,math
A=[json.loads(l) for l in open('/data/articles.jsonl')]
def cos(a,b): return sum(x*y for x,y in zip(a,b))/math.sqrt(sum(x*x for x in a)*sum(y*y for y in b))
random.seed(1); bad=0
for t in range(60):
    q=random.choice(A)['embedding'] if t<30 else [random.gauss(0,1) for _ in range(64)]
    if t<30: q=[x+random.gauss(0,0.02) for x in q]
    s=sorted(A,key=lambda a:-cos(q,a['embedding'])); exp=[a['id'] for a in s[:5]]
    got=subprocess.run(['python3','/app/search.py'],input=json.dumps(q),capture_output=True,text=True).stdout.split()
    if got!=exp: bad+=1; print(t,exp,got,[round(cos(q,a['embedding']),6) for a in s[:6]])
print('mismatches',bad)
"

8  $ redis-cli SAVE; redis-cli LASTSAVE; ls -la /var/lib/redis; redis-cli DBSIZE

{
 "dbsize": 1000,
 "queries": [
  {
   "exit": 0,
   "ids": [
    "kb-0468",
    "kb-0368",
    "kb-0028",
    "kb-0088",
    "kb-0708"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0215",
    "kb-0155",
    "kb-0315",
    "kb-0535",
    "kb-0615"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0963",
    "kb-0123",
    "kb-0423",
    "kb-0583",
    "kb-0823"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0237",
    "kb-0597",
    "kb-0097",
    "kb-0337",
    "kb-0837"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0541",
    "kb-0621",
    "kb-0841",
    "kb-0581",
    "kb-0481"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  }
 ],
 "server": {
  "redis_version": "8.10.2",
  "server_name": null,
  "valkey_version": null
 },
 "server_side_search_commands": [
  "FT.SEARCH"
 ]
}

1  $ redis-cli ping; redis-cli INFO server | grep -E 'redis_version|redis_mode'; redis-cli MODULE LIST; redis-cli FT._LIST

2  $ ls -l /data/articles.jsonl /app /etc/redis/redis.conf; head -c 480 /data/articles.jsonl; echo; python3 - <<'PY'
import importlib.util
for name in ['redis','numpy']:
 print(name,importlib.util.find_spec(name))
PY

3  $ redis-cli CONFIG GET appendonly save dir dbfilename; grep -Ei '^\s*(appendonly|save|dir|dbfilename|loadmodule)' /etc/redis/redis.conf | tail -25

4  $ cat /app/catalog.py; redis-cli FT.CREATE --help 2>&1 | head -20; redis-cli INFO persistence | head -35

5  $ redis-cli FT.CREATE kb_articles ON HASH PREFIX 1 kb:article: SCHEMA id TAG title TEXT embedding VECTOR FLAT 6 TYPE FLOAT64 DIM 64 DISTANCE_METRIC COSINE
python3 - <<'PY'
import json,struct,redis
r=redis.Redis(host='127.0.0.1',port=6379)
count=0
with open('/data/articles.jsonl') as f:
 p=r.pipeline(transaction=False)
 for line in f:
  article=json.loads(line)
  vector=article['embedding']
  assert len(vector)==64 and all(isinstance(x,(int,float)) for x in vector)
  id=article['id']
  p.hset('kb:article:'+str(id),mapping={'id':id,'title':article['title'],'embedding':struct.pack('<64d',*vector)})
  count+=1
  if count%200==0: p.execute()
 p.execute()
print('loaded',count)
assert count==1000
PY
redis-cli FT.INFO kb_articles | head -55

6  $ cat > /app/search.py <<'PY'
"""Print the five closest article IDs for a 64-dimensional query vector."""
import json
import struct
import sys

import redis


def main():
    vector = json.load(sys.stdin)
    if not isinstance(vector, list) or len(vector) != 64:
        raise ValueError('Expected a JSON list of 64 numbers')
    blob = struct.pack('<64d', *vector)

    client = redis.Redis(host='127.0.0.1', port=6379)
    result = client.execute_command(
        'FT.SEARCH', 'kb_articles', '*=>[KNN 5 @embedding $query AS distance]',
        'PARAMS', 2, 'query', blob,
        'SORTBY', 'distance', 'ASC',
        'RETURN', 1, 'id',
        'LIMIT', 0, 5,
        'DIALECT', 2,
    )
    for fields in result[2::2]:
        values = dict(zip(fields[::2], fields[1::2]))
        print(values[b'id'].decode('utf-8'))


if __name__ == '__main__':
    main()
PY
python3 - <<'PY'
import json,subprocess,redis,math
with open('/data/articles.jsonl') as f: articles=[json.loads(s) for s in f]
for q in [articles[0]['embedding'],articles[124]['embedding'], [0.01+i/1000 for i in range(64)]]:
 output=subprocess.check_output(['python3','/app/search.py'],input=json.dumps(q).encode()).decode().splitlines()
 def sim(a):
  v=a['embedding']
  return sum(x*y for x,y in zip(v,q))/math.sqrt(sum(x*x for x in v)*sum(y*y for y in q))
 expected=[a['id'] for a in sorted(articles,key=sim,reverse=True)[:5]]
 print('got:',output,'expected:',expected)
 assert output==expected
PY
redis-cli SAVE; redis-cli DBSIZE; redis-cli FT.INFO kb_articles | grep -A1 num_docs

7  $ python3 - <<'PY'
import redis,json,struct
q=json.loads(open('/data/articles.jsonl').readline())['embedding']
r=redis.Redis()
args=('FT.SEARCH','kb_articles','*=>[KNN 5 @embedding $query AS distance]','PARAMS',2,'query',struct.pack('<64d',*q),'SORTBY','distance','ASC','RETURN',1,'id','LIMIT',0,5,'DIALECT',2)
print(repr(r.execute_command(*args)))
print(repr(r.execute_command(*args, NEVER_DECODE=True)))
PY

8  $ python3 - <<'PY'
p='/app/search.py'
s=open(p).read().replace("    for fields in result[2::2]:\n        values = dict(zip(fields[::2], fields[1::2]))\n        print(values[b'id'].decode('utf-8'))", "    if isinstance(result, dict):\n        for hit in result[b'results']:\n            print(hit[b'extra_attributes'][b'id'].decode('utf-8'))\n    else:\n        for fields in result[2::2]:\n            values = dict(zip(fields[::2], fields[1::2]))\n            print(values[b'id'].decode('utf-8'))")
open(p,'w').write(s)
PY
python3 - <<'PY'
import json,subprocess,math
with open('/data/articles.jsonl') as f: articles=[json.loads(s) for s in f]
for q in [articles[0]['embedding'],articles[124]['embedding'], [0.01+i/1000 for i in range(64)]]:
 output=subprocess.check_output(['python3','/app/search.py'],input=json.dumps(q).encode()).decode().splitlines()
 def sim(a):
  v=a['embedding']
  return sum(x*y for x,y in zip(v,q))/math.sqrt(sum(x*x for x in v)*sum(y*y for y in q))
 expected=[a['id'] for a in sorted(articles,key=sim,reverse=True)[:5]]
 print('got:',output,'expected:',expected)
 assert output==expected
PY

{
 "dbsize": 1000,
 "queries": [
  {
   "exit": 0,
   "ids": [
    "kb-0468",
    "kb-0368",
    "kb-0028",
    "kb-0088",
    "kb-0708"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0215",
    "kb-0155",
    "kb-0315",
    "kb-0535",
    "kb-0615"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0963",
    "kb-0123",
    "kb-0423",
    "kb-0583",
    "kb-0823"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0237",
    "kb-0597",
    "kb-0097",
    "kb-0337",
    "kb-0837"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0541",
    "kb-0621",
    "kb-0841",
    "kb-0581",
    "kb-0481"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  }
 ],
 "server": {
  "redis_version": "8.10.2",
  "server_name": null,
  "valkey_version": null
 },
 "server_side_search_commands": [
  "FT.SEARCH"
 ]
}

1  $ redis-cli ping; redis-cli INFO server | head -25; redis-cli MODULE LIST; ls -la /app /data; head -c 400 /data/articles.jsonl

2  $ python3 - <<'PY'
try: import redis; print('redis-py',redis.__version__)
except Exception as e: print(e)
PY
redis-cli CONFIG GET appendonly save; redis-cli COMMAND INFO FT.CREATE FT.SEARCH | head -80

3  $ cat > /app/search.py <<'PY'
#!/usr/bin/env python3
"""Print the five nearest knowledge-base IDs using Redis cosine vector search."""
import json
import math
import struct
import sys

import redis


def main():
    embedding = json.load(sys.stdin)
    if (not isinstance(embedding, list) or len(embedding) != 64
            or any(isinstance(x, bool) or not isinstance(x, (float, int))
                   or not math.isfinite(x) for x in embedding)):
        raise ValueError('Expected a JSON list of 64 finite numbers')

    vector = struct.pack('<64d', *embedding)
    client = redis.Redis(host='127.0.0.1', port=6379)
    result = client.execute_command(
        'FT.SEARCH', 'idx:kb:articles',
        '*=>[KNN 5 @embedding $query AS distance]',
        'PARAMS', 2, 'query', vector,
        'SORTBY', 'distance', 'ASC', 'RETURN', 1, 'id',
        'LIMIT', 0, 5, 'DIALECT', 2,
    )
    # FT.SEARCH returns [total, key, [field, value, ...], ...].
    for fields in result[2::2]:
        values = dict(zip(fields[::2], fields[1::2]))
        print(values[b'id'].decode('utf-8'))


if __name__ == '__main__':
    main()
PY
python3 - <<'PY'
import json, struct, redis
r = redis.Redis(host='127.0.0.1',port=6379)
try:
    r.execute_command('FT.CREATE', 'idx:kb:articles', 'ON', 'HASH', 'PREFIX', 1, 'kb:article:', 'SCHEMA', 'id', 'TAG', 'title', 'TEXT', 'embedding', 'VECTOR', 'FLAT', 6, 'TYPE', 'FLOAT64', 'DIM', 64, 'DISTANCE_METRIC', 'COSINE')
except redis.ResponseError as e:
    if 'Index already exists' not in str(e): raise
pipeline = r.pipeline(transaction=False)
count = 0
with open('/data/articles.jsonl') as f:
    for line in f:
        doc = json.loads(line)
        assert len(doc['embedding']) == 64
        pipeline.hset('kb:article:' + doc['id'], mapping={
            'id': doc['id'], 'title': doc['title'],
            'embedding': struct.pack('<64d', *doc['embedding'])})
        count += 1
        if count % 200 == 0:
            pipeline.execute()
if count % 200: pipeline.execute()
print('loaded', count)
print('index count:',r.execute_command('FT.SEARCH','idx:kb:articles','*','LIMIT',0,0)[0])
PY

4  $ python3 - <<'PY'
import redis,json
r=redis.Redis(); print(repr(r.execute_command('FT.SEARCH','idx:kb:articles','*','LIMIT',0,1))[:1000]);print(repr(r.execute_command('FT.INFO','idx:kb:articles'))[:1800]);
with open('/data/articles.jsonl') as f: e=json.loads(next(f))['embedding']
import subprocess
p=subprocess.run(['python3','/app/search.py'],input=json.dumps(e),text=True,capture_output=True); print('test',p.returncode,p.stdout,p.stderr)
PY

5  $ python3 - <<'PY'
import redis, json,struct
r=redis.Redis(); v=json.loads(open('/data/articles.jsonl').readline())['embedding']; res=r.execute_command('FT.SEARCH','idx:kb:articles','*=>[KNN 5 @embedding $query AS distance]','PARAMS',2,'query',struct.pack('<64d',*v),'SORTBY','distance','ASC','RETURN',1,'id','LIMIT',0,5,'DIALECT',2); print(res)
PY

6  $ python3 - <<'PY'
p='/app/search.py'
s=open(p).read();s=s.replace('    # FT.SEARCH returns [total, key, [field, value, ...], ...].\n    for fields in result[2::2]:\n        values = dict(zip(fields[::2], fields[1::2]))\n        print(values[b\'id\'].decode(\'utf-8\'))','    # Redis 8 RESP2 returns a map containing the ordered result documents.\n    for document in result[b\'results\']:\n        print(document[b\'extra_attributes\'][b\'id\'].decode(\'utf-8\'))');open(p,'w').write(s)
PY
python3 - <<'PY'
import json,subprocess
with open('/data/articles.jsonl') as f: v=json.loads(next(f))['embedding']
p=subprocess.run(['python3','/app/search.py'], input=json.dumps(v), text=True, capture_output=True)
print('return code:',p.returncode,'output:\n'+p.stdout,'stderr:',p.stderr)
PY
redis-cli BGSAVE; redis-cli INFO persistence | grep -E 'rdb_last_bgsave_status|rdb_bgsave_in_progress'; ls -l /var/lib/redis

7  $ sleep 1; redis-cli INFO persistence | grep -E 'rdb_last_bgsave_status|rdb_bgsave_in_progress'; redis-cli CONFIG GET dir dbfilename; ls -lh /var/lib/redis/dump.rdb; redis-cli FT.INFO idx:kb:articles | head -c 450

{
 "dbsize": 1000,
 "queries": [
  {
   "exit": 0,
   "ids": [
    "kb-0468",
    "kb-0368",
    "kb-0028",
    "kb-0088",
    "kb-0708"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0215",
    "kb-0155",
    "kb-0315",
    "kb-0535",
    "kb-0615"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0963",
    "kb-0123",
    "kb-0423",
    "kb-0583",
    "kb-0823"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0237",
    "kb-0597",
    "kb-0097",
    "kb-0337",
    "kb-0837"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0541",
    "kb-0621",
    "kb-0841",
    "kb-0581",
    "kb-0481"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  }
 ],
 "server": {
  "redis_version": "8.10.2",
  "server_name": null,
  "valkey_version": null
 },
 "server_side_search_commands": [
  "FT.SEARCH"
 ]
}

1  $ redis-cli PING; redis-cli MODULE LIST; redis-cli INFO server | head -25; ls -l /app /data/articles.jsonl

2  $ head -c 900 /data/articles.jsonl; echo; python3 - <<'PY'
try: import redis; print('redis-py',redis.__version__)
except Exception as e: print(type(e).__name__,e)
PY

3  $ cat /app/catalog.py; grep -E '^(save |appendonly |dir |dbfilename |loadmodule )' /etc/redis/redis.conf; redis-cli FT._LIST; redis-cli DBSIZE; id; ls -ld /app

4  $ redis-cli FT.CREATE kb_articles ON HASH PREFIX 1 'kb:article:' SCHEMA id TAG title TEXT embedding VECTOR FLAT 6 TYPE FLOAT64 DIM 64 DISTANCE_METRIC COSINE; python3 - <<'PY'
import json,struct,redis
r=redis.Redis(host='127.0.0.1',port=6379)
p=r.pipeline(transaction=False)
count=0
with open('/data/articles.jsonl') as f:
 for line in f:
  article=json.loads(line)
  assert len(article['embedding'])==64
  p.hset('kb:article:'+article['id'], mapping={'id':article['id'],'title':article['title'],'embedding':struct.pack('<64d',*article['embedding'])})
  count+=1
  if count%200==0: p.execute()
p.execute()
print('loaded',count)
print('indexed',r.execute_command('FT.INFO','kb_articles')[r.execute_command('FT.INFO','kb_articles').index(b'num_docs')+1])
PY

5  $ python3 - <<'PY'
import redis,struct
r=redis.Redis()
print('dbsize',r.dbsize()); print('info',r.execute_command('FT.INFO','kb_articles').get('num_docs'))
q=r.hget('kb:article:kb-0001','embedding')
for cmd in [( 'FT.SEARCH','kb_articles','*=>[KNN 5 @embedding $vec AS distance]','PARAMS','2','vec',q,'SORTBY','distance','ASC','RETURN','1','id','DIALECT','2'),('FT.SEARCH','kb_articles','*=>[KNN 5 @embedding $vec AS distance]','PARAMS','2','vec',q,'SORTBY','distance','ASC','RETURN','2','id','distance','DIALECT','2')]:
 try: print(r.execute_command(*cmd))
 except Exception as e: print(repr(e))
PY

6  $ cat > /app/search.py <<'PY'
#!/usr/bin/env python3
"""Find the five closest knowledge-base articles using Redis vector search."""
import json
import struct
import sys

import redis


def main():
    vector = json.load(sys.stdin)
    if not isinstance(vector, list) or len(vector) != 64:
        raise ValueError('expected a JSON list of 64 numbers')
    query = struct.pack('<64d', *vector)
    connection = redis.Redis(host='127.0.0.1', port=6379)
    response = connection.execute_command(
        'FT.SEARCH', 'kb_articles',
        '*=>[KNN 5 @embedding $query_vector AS distance]',
        'PARAMS', '2', 'query_vector', query,
        'SORTBY', 'distance', 'ASC',
        'RETURN', '1', 'id',
        'DIALECT', '2',
    )
    for result in response['results']:
        print(result['extra_attributes']['id'].decode('utf-8'))


if __name__ == '__main__':
    main()
PY
python3 - <<'PY'
import json,subprocess
with open('/data/articles.jsonl') as f: embedding=json.loads(next(f))['embedding']
print(subprocess.check_output(['python3','/app/search.py'],input=json.dumps(embedding).encode()).decode())
PY
redis-cli --raw FT.INFO kb_articles | grep -A1 num_docs | head; redis-cli CONFIG GET save appendonly; redis-cli SAVE; ls -l /var/lib/redis | head

7  $ python3 - <<'PY'
import redis,struct
r=redis.Redis(); v=r.hget('kb:article:kb-0001','embedding'); x=r.execute_command('FT.SEARCH','kb_articles','*=>[KNN 5 @embedding $query_vector AS distance]','PARAMS','2','query_vector',v,'SORTBY','distance','ASC','RETURN','1','id','DIALECT','2');print(type(x),repr(x))
PY

8  $ sed -i "s/response\['results'\]/response[b'results']/; s/\['extra_attributes'\]\['id'\]/[b'extra_attributes'][b'id']/" /app/search.py
python3 - <<'PY'
import json,subprocess
with open('/data/articles.jsonl') as f: embedding=json.loads(next(f))['embedding']
print(subprocess.check_output(['python3','/app/search.py'],input=json.dumps(embedding).encode()).decode(),end='')
PY

{
 "dbsize": 1000,
 "queries": [
  {
   "exit": 0,
   "ids": [
    "kb-0468",
    "kb-0368",
    "kb-0028",
    "kb-0088",
    "kb-0708"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0215",
    "kb-0155",
    "kb-0315",
    "kb-0535",
    "kb-0615"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0963",
    "kb-0123",
    "kb-0423",
    "kb-0583",
    "kb-0823"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0237",
    "kb-0597",
    "kb-0097",
    "kb-0337",
    "kb-0837"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0541",
    "kb-0621",
    "kb-0841",
    "kb-0581",
    "kb-0481"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  }
 ],
 "server": {
  "redis_version": "8.10.2",
  "server_name": null,
  "valkey_version": null
 },
 "server_side_search_commands": [
  "FT.SEARCH"
 ]
}

1  $ head -c 600 /data/articles.jsonl; echo; wc -l /data/articles.jsonl; which redis-cli; redis-cli ping; redis-cli MODULE LIST; redis-cli --version

2  $ python3 -c "import redis; print(redis.__version__)" 2>&1; pip list 2>/dev/null | head -50

3  $ python3 -c "
import json
d=json.loads(open('/data/articles.jsonl').readline())
print(len(d['embedding']), sum(x*x for x in d['embedding']))
"; ls /app 2>&1

4  $ cat /app/catalog.py; ls -la /app; redis-cli CONFIG GET * 2>/dev/null | head -5

5  $ cat > /app/load_articles.py <<'EOF'
"""Load /data/articles.jsonl into a Redis vector set for semantic search."""
import json
import redis

VECTOR_SET_KEY = "articles:vectorset"
DATA = "/data/articles.jsonl"


def main() -> None:
    r = redis.Redis(host="127.0.0.1", port=6379, decode_responses=True)
    r.delete(VECTOR_SET_KEY)

    loaded = 0
    pipe = r.pipeline(transaction=False)
    dim = None
    with open(DATA) as fh:
        for line in fh:
            line = line.strip()
            if not line:
                continue
            art = json.loads(line)
            emb = art["embedding"]
            if dim is None:
                dim = len(emb)
            assert len(emb) == dim
            pipe.execute_command(
                "VADD", VECTOR_SET_KEY, "VALUES", dim, *[repr(float(x)) for x in emb],
                art["id"], "SETATTR", json.dumps({"title": art["title"]}),
            )
            loaded += 1
            if loaded % 200 == 0:
                pipe.execute()
    pipe.execute()
    print("loaded", loaded, "elements; card =", r.vcard(VECTOR_SET_KEY) if hasattr(r, "vcard") else r.execute_command("VCARD", VECTOR_SET_KEY),
          "dim =", r.execute_command("VDIM", VECTOR_SET_KEY))


if __name__ == "__main__":
    main()
EOF
python3 /app/load_articles.py

6  $ cat > /app/search.py <<'EOF'
#!/usr/bin/env python3
"""Semantic search over the knowledge-base articles stored in Redis.

Usage:  python3 /app/search.py < query.json

`query.json` contains a single query embedding: a JSON list of 64 floats.
Prints the ids of the 5 most similar articles (by cosine similarity),
one per line, most similar first.

All the data and the similarity computation live in Redis: this script only
forwards the query vector to the `articles:vectorset` vector set with VSIM.
"""
import json
import sys

import redis

VECTOR_SET_KEY = "articles:vectorset"
TOP_K = 5
REDIS_HOST = "127.0.0.1"
REDIS_PORT = 6379


def parse_query(raw):
    """Accept a bare JSON list of floats, or an object wrapping it."""
    data = json.loads(raw)
    if isinstance(data, dict):
        for field in ("embedding", "vector", "query", "query_embedding"):
            if field in data:
                data = data[field]
                break
    if isinstance(data, list) and len(data) == 1 and isinstance(data[0], list):
        data = data[0]
    if not isinstance(data, list):
        raise ValueError("query must be a JSON list of floats")
    return [float(x) for x in data]


def main() -> None:
    raw = sys.stdin.read()
    query = parse_query(raw)
    if not query:
        return

    client = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True)
    args = ["VSIM", VECTOR_SET_KEY, "VALUES", len(query), *query, "COUNT", TOP_K, "TRUTH"]
    result = client.execute_command(*args)

    # VSIM returns Python list submissions (or list of [element, score] pairs).
    idents = []
    if result and isinstance(result[0], (list, tuple)):
        idents = [pair[0] for pair in result]
    else:
        idents = list(result)

    sys.stdout.write("\n".join(str(i) for i in idents) + "\n")


if __name__ == "__main__":
    main()
EOF
python3 -c "
import json
row=json.loads(open('/data/articles.jsonl').readline())
print(json.dumps(row['embedding']))" > /tmp/q.json
python3 /app/search.py < /tmp/q.json

7  $ cat > /tmp/verify.py <<'EOF'
import json, subprocess, sys
import numpy as np

arts = [json.loads(l) for l in open('/data/articles.jsonl')]
ids = [a['id'] for a in arts]
M = np.array([a['embedding'] for a in arts], dtype=np.float64)
M /= np.linalg.norm(M, axis=1, keepdims=True)

rng = np.random.default_rng(7)
bad = 0
for t in range(30):
    if t == 0:
        q = M[0]
    else:
        q = rng.normal(size=64)
        q /= np.linalg.norm(q)
    # brute force cosine
    sims = M @ q
    top = [ids[i] for i in np.argsort(-sims)[:5]]
    out = subprocess.run([sys.executable, '/app/search.py'], input=json.dumps(q.tolist()),
                         capture_output=True, text=True, check=True).stdout.split()
    if out != top:
        bad += 1
        print("MISMATCH", t, out, top)
print("done, mismatches:", bad)
EOF
python3 /tmp/verify.py

8  $ cd /app && python3 - <<'EOF'
import json, numpy as np, redis
r = redis.Redis(decode_responses=True)
arts=[json.loads(l) for l in open('/data/articles.jsonl')]
ids=[a['id'] for a in arts]
M=np.array([a['embedding'] for a in arts]); M/=np.linalg.norm(M,axis=1,keepdims=True)
rng=np.random.default_rng(7)
queries=[M[0]]+[(lambda v: v/np.linalg.norm(v))(rng.normal(size=64)) for _ in range(30)]
for ef in [100, 500, 1000, 5000]:
    bad=0
    for q in queries:
        top=[ids[i] for i in np.argsort(-(M@q))[:5]]
        res=r.execute_command("VSIM","articles:vectorset","VALUES",64,*q.tolist(),"COUNT",5,"EF",ef,"TRUTH")
        out=[p[0] for p in res]
        if out!=top: bad+=1
    print("EF",ef,"mismatches",bad)
EOF

9  $ cd /app && python3 - <<'EOF'
import json, numpy as np, redis
r = redis.Redis(decode_responses=True)
arts=[json.loads(l) for l in open('/data/articles.jsonl')]
ids=[a['id'] for a in arts]
M=np.array([a['embedding'] for a in arts]); M/=np.linalg.norm(M,axis=1,keepdims=True)
q=M[0]
sims=M@q
top=[(ids[i],float(sims[i])) for i in np.argsort(-sims)[:8]]
print("numpy top:",top)
print("VSIM TRUTH:", r.execute_command("VSIM","articles:vectorset","VALUES",64,*q.tolist(),"COUNT",8,"TRUTH","WITHSCORES"))
print("VSIM:", r.execute_command("VSIM","articles:vectorset","VALUES",64,*q.tolist(),"COUNT",8,"WITHSCORES"))
print("norm after:", np.linalg.norm(q), "count of ties at 5th:",sorted(sims)[-6:])
EOF

10  $ cd /app && python3 - <<'EOF'
import json, numpy as np, redis
r = redis.Redis(decode_responses=True)
arts=[json.loads(l) for l in open('/data/articles.jsonl')]
ids=[a['id'] for a in arts]
M=np.array([a['embedding'] for a in arts],dtype=np.float32)
M/=np.linalg.norm(M,axis=1,keepdims=True)
rng=np.random.default_rng(7)
qs=[M[0]]+[(lambda v:(v/np.linalg.norm(v)).astype(np.float32))(rng.normal(size=64)) for _ in range(40)]
for tag,Mq in [("fp32",M)]:
    bad=0
    for q in qs:
        top=[ids[i] for i in np.argsort(-(Mq@q))[:5]]
        res=r.execute_command("VSIM","articles:vectorset","VALUES",64,*[float(x) for x in q],"COUNT",5,"TRUTH")
        out=[p[0] for p in res]
        if out!=top:
            bad+=1
            # show gap
            s=Mq@q; order=np.argsort(-s)
            print("MISMATCH",[ids[i] for i in order[:6]],[float(s[i]) for i in order[:6]])
    print(tag,"mismatches",bad,"/",len(qs))
EOF

11  $ cd /app && sed -i 's/art\["id"\], "SETATTR"/art["id"], "NOQUANT", "SETATTR"/' load_articles.py && grep -n 'VADD' load_articles.py && python3 load_articles.py

12  $ cd /app && sed -n '25,32p' load_articles.py && redis-cli VINFO articles:vectorset | head -20

13  $ cd /app && python3 - <<'EOF'
import json, numpy as np, redis
r = redis.Redis(decode_responses=True)
arts=[json.loads(l) for l in open('/data/articles.jsonl')]
ids=[a['id'] for a in arts]
M=np.array([a['embedding'] for a in arts],dtype=np.float32)
M/=np.linalg.norm(M,axis=1,keepdims=True)
rng=np.random.default_rng(7)
qs=[M[0]]+[(lambda v:(v/np.linalg.norm(v)).astype(np.float32))(rng.normal(size=64)) for _ in range(60)]
bad=0
for q in qs:
    top=[ids[i] for i in np.argsort(-(M@q))[:5]]
    res=r.execute_command("VSIM","articles:vectorset","VALUES",64,*[float(x) for x in q],"COUNT",5,"TRUTH")
    out=[p[0] for p in res]
    if out!=top:
        bad+=1; print("MISMATCH",out,top)
print("mismatches",bad,"/",len(qs))
EOF

14  $ cd /app && redis-cli VSIM articles:vectorset VALUES 2 1 0 COUNT 3; echo ---; redis-cli VSIM articles:vectorset VALUES 2 1 0 COUNT 3 TRUTH; echo ---; python3 -c "
import json,numpy as np,redis
r=redis.Redis(decode_responses=True)
arts=[json.loads(l) for l in open('/data/articles.jsonl')]
M=np.array([a['embedding'] for a in arts],dtype=np.float32); M/=np.linalg.norm(M,axis=1,keepdims=True)
q=M[0]
res=r.execute_command('VSIM','articles:vectorset','VALUES',64,*q.tolist(),'COUNT',3,'TRUTH')
print(repr(res))
"

15  $ cd /app && cat > /tmp/vf.py <<'EOF'
import json, numpy as np, redis
r = redis.Redis(decode_responses=True)
arts=[json.loads(l) for l in open('/data/articles.jsonl')]
ids=[a['id'] for a in arts]
M=np.array([a['embedding'] for a in arts],dtype=np.float32); M/=np.linalg.norm(M,axis=1,keepdims=True)
rng=np.random.default_rng(7)
qs=[M[0]]+[(lambda v:(v/np.linalg.norm(v)).astype(np.float32))(rng.normal(size=64)) for _ in range(80)]
def names(res):
    out=[]
    for x in res:
        out.append(x[0] if isinstance(x,(list,tuple)) else x)
    return out
bad=0
for q in qs:
    top=[ids[i] for i in np.argsort(-(M@q))[:5]]
    out=names(r.execute_command("VSIM","articles:vectorset","VALUES",64,*[float(x) for x in q],"COUNT",5,"TRUTH"))
    if out!=top:
        bad+=1; print("MISMATCH",out,top)
print("mismatches",bad,"/",len(qs))
EOF
python3 /tmp/vf.py

16  $ cd /app && python3 - <<'EOF'
import json, numpy as np, redis
r=redis.Redis(decode_responses=True)
arts=[json.loads(l) for l in open('/data/articles.jsonl')]
ids=[a['id'] for a in arts]
M64=np.array([a['embedding'] for a in arts],dtype=np.float64); M64n=M64/np.linalg.norm(M64,axis=1,keepdims=True)
rng=np.random.default_rng(1)
worst=[]
for t in range(200):
    q=rng.normal(size=64); q=q/np.linalg.norm(q)
    order=np.argsort(-(M64n@q))
    s=(M64n@q)[order[:7]]
    gap=s[4]-s[5]
    worst.append((gap, ids[order[4]], ids[order[5]]))
worst.sort()
print("smallest top5/top6 gaps (f64):")
for g,a,b in worst[:8]: print(f"  {g:.2e} {a} vs {b}")
print("median gap:", np.median([w[0] for w in worst]))
EOF

17  $ cd /app && python3 - <<'EOF'
import json, numpy as np, redis
r=redis.Redis(decode_responses=True)
arts=[json.loads(l) for l in open('/data/articles.jsonl')]
ids=[a['id'] for a in arts]
M=np.array([a['embedding'] for a in arts],dtype=np.float64); M/=np.linalg.norm(M,axis=1,keepdims=True)
rng=np.random.default_rng(3)
def names(res): return [x[0] if isinstance(x,(list,tuple)) else x for x in res]
bad=0
for t in range(300):
    q=rng.normal(size=64); q/=np.linalg.norm(q)
    top=[ids[i] for i in np.argsort(-(M@q))[:5]]
    out=names(r.execute_command("VSIM","articles:vectorset","VALUES",64,*q.tolist(),"COUNT",5,"TRUTH"))
    if out!=top: bad+=1; print("MISMATCH",t,out,top)
print("f64-baseline mismatches:",bad,"/300")
EOF

18  $ cat > /app/search.py <<'EOF'
#!/usr/bin/env python3
"""Semantic search over the knowledge-base articles stored in Redis.

Usage:
    python3 /app/search.py < query.json

`query.json` holds a single query embedding: a JSON list of 64 floats.

The script prints the ids of the 5 most similar articles (cosine
similarity), one per line, most similar first.

All articles and the similarity computation live inside Redis: the
embeddings are stored in the `articles:vectorset` vector set and the
ranking is delegated to the Redis `VSIM` command.  This script never
reads /data/articles.jsonl and never computes similarities itself.
"""
import json
import os
import sys

import redis

VECTOR_SET_KEY = "articles:vectorset"
TOP_K = 5
HOST = os.environ.get("REDIS_HOST", "127.0.0.1")
PORT = int(os.environ.get("REDIS_PORT", "6379"))


def parse_query(raw):
    """Return the query embedding from the raw stdin text.

    Accepts a bare JSON list of floats as well as a JSON object that wraps
    the embedding under a common field name.
    """
    data = json.loads(raw)
    if isinstance(data, dict):
        for field in ("embedding", "vector", "query", "query_embedding"):
            if field in data:
                data = data[field]
                break
    # Tolerate a single-element list wrapping the vector.
    if isinstance(data, list) and len(data) == 1 and isinstance(data[0], list):
        data = data[0]
    if not isinstance(data, list):
        raise ValueError("query must be a JSON list of floats")
    return [float(x) for x in data]


def main():
    raw = sys.stdin.read()
    query = parse_query(raw)
    if not query:
        return

    client = redis.Redis(host=HOST, port=PORT, decode_responses=True)

    # VSIM runs the similarity search (HNSW / exact) inside Redis.
    # TRUTH forces an exact result, which is cheap for this dataset size.
    reply = client.execute_command(
        "VSIM", VECTOR_SET_KEY, "VALUES", len(query), *query,
        "COUNT", TOP_K, "TRUTH",
    )

    # The reply is a list of element names (optionally [name, score] pairs).
    idents = [
        item[0] if isinstance(item, (list, tuple)) else item
        for item in reply
    ]

    sys.stdout.write("\n".join(str(i) for i in idents) + "\n")


if __name__ == "__main__":
    main()
EOF
cd /app && python3 -c "
import json,random
random.seed(99)
print(json.dumps([round(random.uniform(-1,1),6) for _ in range(64)]))" > /tmp/q2.json
cat /tmp/q2.json
echo "--- search output:"
python3 /app/search.py < /tmp/q2.json
echo "--- titles of hits:"
python3 /app/search.py < /tmp/q2.json | while read id; do redis-cli VGETATTR articles:vectorset "$id"; done

19  $ redis-cli CONFIG GET save; redis-cli CONFIG GET appendonly; redis-cli TTL articles:vectorset; redis-cli TYPE articles:vectorset; redis-cli VCARD articles:vectorset; redis-cli DBSIZE

20  $ grep -E '^(dir|dbfilename|appendonly|save) ' /etc/redis/redis.conf; echo ---; redis-cli CONFIG GET dir; redis-cli CONFIG GET dbfilename; redis-cli BGSAVE; sleep 2; ls -la $(redis-cli CONFIG GET dir | tail -1)

21  $ cd /tmp && python3 /app/search.py < /tmp/q2.json; echo "exit=$?"; echo "--- reload resilience test (flush then restore from RDB) ---"; redis-cli INFO persistence | grep -E 'rdb_last_bgsave_status|loading'

22  $ redis-cli SHUTDOWN NOSAVE 2>/dev/null; sleep 2; redis-server /etc/redis/redis.conf --daemonize yes; sleep 2; redis-cli PING; redis-cli VCARD articles:vectorset; cd /tmp && python3 /app/search.py < /tmp/q2.json

23  $ echo "Redis keys:"; redis-cli KEYS '*'; echo; echo "Vector set info:"; redis-cli VINFO articles:vectorset; echo; echo "Sample stored vector + attribute:"; redis-cli VGETATTR articles:vectorset kb-0001; echo; echo "search.py head:"; head -30 /app/search.py

{
 "dbsize": 1,
 "queries": [
  {
   "exit": 0,
   "ids": [
    "kb-0468",
    "kb-0368",
    "kb-0028",
    "kb-0088",
    "kb-0708"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0215",
    "kb-0155",
    "kb-0315",
    "kb-0535",
    "kb-0615"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0963",
    "kb-0123",
    "kb-0423",
    "kb-0583",
    "kb-0823"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0237",
    "kb-0597",
    "kb-0097",
    "kb-0337",
    "kb-0837"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0541",
    "kb-0621",
    "kb-0841",
    "kb-0581",
    "kb-0481"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  }
 ],
 "server": {
  "redis_version": "8.10.2",
  "server_name": null,
  "valkey_version": null
 },
 "server_side_search_commands": [
  "VSIM"
 ]
}

1  $ ls -la /data/ /app/ 2>&1; echo "---"; head -c 500 /data/articles.jsonl; echo; echo "---"; wc -l /data/articles.jsonl

2  $ redis-cli -h 127.0.0.1 -p 6379 ping; redis-cli -h 127.0.0.1 -p 6379 info server | head -30; echo "---modules---"; redis-cli -h 127.0.0.1 -p 6379 module list

3  $ cat /app/catalog.py; echo "---"; python3 -c "import redis; print(redis.__version__)" 2>&1; pip list 2>/dev/null | grep -i redis

4  $ redis-cli VADD vs_test VALUES 2 1.0 1.0 a; redis-cli VADD vs_test VALUES 2 -1.0 -1.0 b; redis-cli VSIM vs_test VALUES 2 1.0 0.9 WITHSCORES; echo "---default---"; redis-cli VSIM vs_test VALUES 2 1.0 0.9; redis-cli VINFO vs_test; redis-cli DEL vs_test

5  $ python3 - <<'EOF'
import json, statistics
rows=[json.loads(l) for l in open('/data/articles.jsonl')]
print(len(rows))
e=rows[0]
print(e['id'], e['title'], len(e['embedding']))
import math
dims=set(len(r['embedding']) for r in rows)
print("dims",dims)
ids=[r['id'] for r in rows]
print("unique ids", len(set(ids)), ids[:3], ids[-3:])
norms=[math.sqrt(sum(x*x for x in r['embedding'])) for r in rows]
print("norm min/max/mean", min(norms), max(norms), statistics.mean(norms))
EOF

6  $ mkdir -p /app && cat > /app/load_articles.py <<'EOF'
import json
import redis

VSET_KEY = "articles:vset"
HASH_KEY = "articles:meta"
BATCH = 500

r = redis.Redis(host="127.0.0.1", port=6379, decode_responses=True)

rows = []
with open("/data/articles.jsonl") as f:
    for line in f:
        line = line.strip()
        if line:
            rows.append(json.loads(line))

r.delete(VSET_KEY, HASH_KEY)

def batches(seq, n):
    for i in range(0, len(seq), n):
        yield seq[i:i + n]

for chunk in batches(rows, BATCH):
    p = r.pipeline(transaction=False)
    for a in chunk:
        vec = [float(x) for x in a["embedding"]]
        elem = a["id"]
        attrs = json.dumps({"id": a["id"], "title": a["title"]})
        p.execute_command(
            "VADD", VSET_KEY, "VALUES", len(vec), *[repr(v) for v in vec],
            elem, "NOQUANT", "SETATTR", attrs,
        )
        p.hset(HASH_KEY, elem, json.dumps(a))
    p.execute()

print("loaded:", r.execute_command("VCARD", VSET_KEY))
print("dim:", r.execute_command("VDIM", VSET_KEY))
print("meta fields:", r.hlen(HASH_KEY))
EOF
python3 /app/load_articles.py

7  $ cat > /app/search.py <<'EOF'
#!/usr/bin/env python3
"""Semantic search over the knowledge-base articles stored in Redis.

Usage:  python3 /app/search.py < query.json

Reads a single query embedding (JSON list of 64 floats) from stdin at most once,
then asks Redis (via the vector-set VSIM command) for the 5 most similar
articles by cosine similarity and prints their ids, most similar first.

All stored vectors and similarity computation live inside Redis; this script
never reads /data/articles.jsonl and never computes similarities itself.
"""
import json
import sys

import redis

VSET_KEY = "articles:vset"
TOP_K = 5


def read_query_vector():
    data = sys.stdin.read().strip()
    if not data:
        sys.exit("no query embedding on stdin")
    vec = json.loads(data)
    # Tolerate either a bare list or a single-element list of lists.
    if vec and isinstance(vec[0], list):
        vec = vec[0]
    if not isinstance(vec, list) or not vec:
        sys.exit("query embedding must be a non-empty JSON list of floats")
    return [float(x) for x in vec]


def main():
    vector = read_query_vector()

    r = redis.Redis(host="127.0.0.1", port=6379, decode_responses=True)

    # VSIM runs the similarity search inside Redis; default metric is cosine.
    result = r.execute_command(
        "VSIM", VSET_KEY, "VALUES", len(vector),
        *[repr(x) for x in vector],
        "COUNT", TOP_K,
    )

    for elem in result[:TOP_K]:
        print(elem)


if __name__ == "__main__":
    main()
EOF
chmod +x /app/search.py
python3 -c "
import json
q=json.loads(open('/data/articles.jsonl').readline())['embedding']
open('/tmp/q.json','w').write(json.dumps(q))
"
python3 /app/search.py < /tmp/q.json

8  $ cat > /tmp/verify.py <<'EOF'
import json, math, subprocess, random

rows=[json.loads(l) for l in open('/data/articles.jsonl')]
def cos(a,b):
    return sum(x*y for x,y in zip(a,b))/ (math.sqrt(sum(x*x for x in a))*math.sqrt(sum(y*y for y in b)))

random.seed(1)
test_ids = random.sample(range(1000), 15) + [0]
ok=0
for i in test_ids:
    q=rows[i]['embedding']
    expected=[r['id'] for r in sorted(rows,key=lambda r: -cos(q,r['embedding']))[:5]]
    out=subprocess.run(['python3','/app/search.py'],input=json.dumps(q),capture_output=True,text=True).stdout.split()
    status = "OK" if out==expected else "MISMATCH"
    if out==expected: ok+=1
    print(status, rows[i]['id'], out, expected if status!="OK" else "")
print(f"{ok}/{len(test_ids)} exact matches")
EOF
python3 /tmp/verify.py

9  $ grep -n "save\|appendonly\|dir " /etc/redis/redis.conf | grep -v "^#" | head -20; echo "---keyspace---"; redis-cli dbsize; redis-cli type articles:vset; redis-cli type articles:meta

10  $ grep -n "^save\|^appendonly\|^dbfilename" /etc/redis/redis.conf; echo "---"; redis-cli SAVE; ls -la /var/lib/redis/; echo "---INFO persistence---"; redis-cli info persistence | grep -E "rdb_last_bgsave_status|aof_enabled|rdb_changes"

11  $ redis-cli VGETATTR articles:vset kb-0001; echo "---"; redis-cli HGET articles:meta kb-0001; echo "--- test 2nd query ---"; python3 -c "
import json
rows=[json.loads(l) for l in open('/data/articles.jsonl')]
open('/tmp/q2.json','w').write(json.dumps(rows[500]['embedding']))
"; python3 /app/search.py < /tmp/q2.json; echo "--- empty stdin ---"; printf '' | python3 /app/search.py; echo "exit=$?"

12  $ cat /proc/$(pgrep -x redis-server)/cmdline | tr '\0' ' '; echo; echo "---restart---"; redis-cli SHUTDOWN NOSAVE 2>/dev/null; sleep 2; 
# restart the server with the same config
/root/redis-server /etc/redis/redis.conf --daemonize yes 2>&1; sleep 2
redis-cli ping; redis-cli VCARD articles:vset; redis-cli HGET articles:meta kb-0001; echo "--- search after restart ---"; python3 /app/search.py < /tmp/q.json

13  $ which redis-server; find / -name "redis-server" -type f 2>/dev/null | head; echo "---supervisor/systemd---"; ls /etc/supervisor/conf.d/ 2>/dev/null; ls /etc/systemd/system/ 2>/dev/null | head; cat /proc/1/cmdline | tr '\0' ' '; echo; ps aux | head -20

14  $ /usr/local/bin/redis-server /etc/redis/redis.conf --daemonize yes; sleep 2; redis-cli ping; echo "--- modules ---"; redis-cli module list | grep -A1 name | head; echo "--- data survived? ---"; redis-cli dbsize; redis-cli VCARD articles:vset; redis-cli HGET articles:meta kb-0001

15  $ python3 /app/search.py < /tmp/q.json; echo "--- full re-verify ---"; python3 /tmp/verify.py | tail -3; echo "--- VSIM with scores sanity ---"; redis-cli VSIM articles:vset ELE kb-0001 WITHSCORES COUNT 5 | head -12

16  $ redis-cli SAVE; ls -la /var/lib/redis/; echo "--- final keys ---"; redis-cli --scan; echo "--- search.py has no file access to jsonl ---"; grep -n "articles.jsonl\|open(" /app/search.py || echo "(none)"; echo "--- final check ---"; python3 /app/search.py < /tmp/q2.json

{
 "dbsize": 2,
 "queries": [
  {
   "exit": 0,
   "ids": [
    "kb-0468",
    "kb-0368",
    "kb-0028",
    "kb-0088",
    "kb-0708"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0215",
    "kb-0155",
    "kb-0315",
    "kb-0535",
    "kb-0615"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0963",
    "kb-0123",
    "kb-0423",
    "kb-0583",
    "kb-0823"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0237",
    "kb-0597",
    "kb-0097",
    "kb-0337",
    "kb-0837"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0541",
    "kb-0621",
    "kb-0841",
    "kb-0581",
    "kb-0481"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  }
 ],
 "server": {
  "redis_version": "8.10.2",
  "server_name": null,
  "valkey_version": null
 },
 "server_side_search_commands": [
  "VSIM"
 ]
}

1  $ head -c 600 /data/articles.jsonl; echo; wc -l /data/articles.jsonl

2  $ redis-cli -h 127.0.0.1 -p 6379 ping; redis-cli info server | head -20

3  $ redis-cli command info VADD; redis-cli command info VSIM; redis-cli command info HSET

4  $ python3 -c "import redis, sys; print(redis.__version__)"; pip list 2>/dev/null | grep -i -E "redis|numpy"; python3 -c "import numpy; print(numpy.__version__)"

5  $ redis-cli -h 127.0.0.1 -p 6379 DEL t1 >/dev/null
redis-cli VADD t1 VALUES 2 1 1 A
redis-cli VADD t1 VALUES 2 -1 -1 B
redis-cli VADD t1 VALUES 2 1 0 C
redis-cli VADD t1 VALUES 2 0 1 D
redis-cli VSIM t1 VALUES 2 1 1 WITHSCORES COUNT 3
echo "--- truth ---"
redis-cli VSIM t1 VALUES 2 1 1 WITHSCORES COUNT 3 TRUTH
echo "--- info ---"
redis-cli VINFO t1

6  $ redis-cli DEL t2 >/dev/null
redis-cli VADD t2 VALUES 2 100 100 A NOQUANT
redis-cli VADD t2 VALUES 2 -5 -5 B NOQUANT
redis-cli VADD t2 VALUES 2 1 0.2 C NOQUANT
redis-cli VSIM t2 VALUES 2 1 1 WITHSCORES COUNT 3
echo "--- reduced query ---"
redis-cli VSIM t2 VALUES 2 0.5 0.5 WITHSCORES COUNT 3
echo "--- zero query ---"
redis-cli VSIM t2 VALUES 2 0 0 WITHSCORES COUNT 3
echo "--- self ---"
redis-cli VSIM t2 ELE A WITHSCORES COUNT 3
redis-cli DEL t2 >/dev/null

7  $ grep -vE '^\s*#|^\s*$' /etc/redis/redis.conf | head -60; echo "=== dir ==="; redis-cli config get dir; redis-cli config get save; redis-cli config get appendonly

8  $ ls -la /data; python3 -c "
import json
embs=[]
ids=[]
with open('/data/articles.jsonl') as f:
    for line in f:
        o=json.loads(line)
        ids.append(o['id']); embs.append(o['embedding'])
print(len(ids), len(set(ids)))
print(set(len(e) for e in embs))
import math
norms=[math.sqrt(sum(x*x for x in e)) for e in embs]
print(min(norms), max(norms))
"

9  $ mkdir -p /app && cat > /app/_load_test.py <<'EOF'
import json, math, numpy as np, redis

recs=[json.loads(l) for l in open('/data/articles.jsonl')]
ids=[r['id'] for r in recs]
X=np.array([r['embedding'] for r in recs], dtype=np.float64)
X/=np.linalg.norm(X,axis=1,keepdims=True)

r=redis.Redis(host='127.0.0.1',port=6379,decode_responses=True)

def build(key, extra):
    r.delete(key)
    p=r.pipeline(transaction=False)
    for i,(aid,v) in enumerate(zip(ids,X)):
        args=['VADD',key]+extra+[aid]+[repr(float(x)) for x in v]
        p.execute_command(*args)
        if i%500==499: p.execute()
    p.execute()

build('tv_quant',['VALUES','64'])
build('tv_noquant',['VALUES','64','NOQUANT'])
print('built', r.vcard('tv_quant'), r.vcard('tv_noquant'))

rng=np.random.default_rng(7)
mismatch_q=mismatch_n=0
for t in range(300):
    q=rng.normal(size=64); q/=np.linalg.norm(q)
    exact=np.argsort(-(X@q))[:5]
    exp=[ids[i] for i in exact]
    gotq=r.execute_command('VSIM','tv_quant','VALUES','64',*[repr(float(x)) for x in q],'COUNT',5)
    gotn=r.execute_command('VSIM','tv_noquant','VALUES','64',*[repr(float(x)) for x in q],'COUNT',5)
    if list(gotq)!=exp: mismatch_q+=1
    if list(gotn)!=exp: mismatch_n+=1
print('top5 mismatch quant:',mismatch_q,'/300   noquant:',mismatch_n,'/300')
EOF
python3 /app/_load_test.py

10  $ cd /app && sed -i "s/args=\['VADD',key\]+extra+\[aid\]+\[repr(float(x)) for x in v\]/args=['VADD',key]+extra+[aid]+[repr(float(x)) for x in v]/" _load_test.py && python3 - <<'EOF'
# just rewrite properly
src=open('/app/_load_test.py').read()
print('VALUES','64' in src)
EOF
grep -n "args=" /app/_load_test.py

11  $ redis-cli DEL dbg >/dev/null; redis-cli VADD dbg VALUES 64 kb-0001 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 0.1
echo "exit=$?"
redis-cli -3 2>/dev/null; python3 -c "
import redis
r=redis.Redis(decode_responses=True)
print(r.execute_command('DEL','dbg'))
try:
    print(r.execute_command('VADD','dbg','VALUES','64','kb-0001',*[str(0.1)]*64))
except Exception as e:
    print('ERR', e)
"

12  $ python3 - <<'EOF'
import redis
r=redis.Redis(decode_responses=True)
r.delete('dbg')
v=[0.1]*64
try:
    print('VALUES 64:', r.execute_command('VADD','dbg','VALUES','64','kb-0001',*[repr(x) for x in v]))
except Exception as e: print('ERR a', e)
r.delete('dbg')
try:
    print('VALUES omits num? ', r.execute_command('VADD','dbg','VALUES','kb-0001',*[repr(x) for x in v]))
except Exception as e: print('ERR b', e)
# try 3-dim
r.delete('dbg')
try:
    print('3 dim:', r.execute_command('VADD','dbg','VALUES','3','x','0.1','0.2','0.3'))
except Exception as e: print('ERR c', e)
try:
    print('3 dim again:', r.execute_command('VADD','dbg','VALUES','3','y','0.1','0.2','0.3'))
except Exception as e: print('ERR d', e)
EOF
echo "--- CLI count check ---"
redis-cli VADD dbg VALUES 3 z 0.1 0.2 0.3; echo "exit=$?"

13  $ redis-cli DEL o1 >/dev/null
echo "attempt: VADD o1 VALUES 2 <vec 0.6 0.8> <element E1>"; redis-cli VADD o1 VALUES 2 0.6 0.8 E1; echo "rc=$?"
redis-cli VINFO o1
echo "--- vsim ele E1 ---"; redis-cli VSIM o1 ELE E1 WITHSCORES
echo "--- vemb E1 ---"; redis-cli VEMB o1 E1
echo; echo "attempt documented order: VADD o2 VALUES 2 E2 0.6 0.8"; redis-cli DEL o2>/dev/null; redis-cli VADD o2 VALUES 2 E2 0.6 0.8; echo "rc=$?"

14  $ cd /app && cat > _load_test.py <<'EOF'
import json, numpy as np, redis, time

recs=[json.loads(l) for l in open('/data/articles.jsonl')]
ids=[r['id'] for r in recs]
X=np.array([r['embedding'] for r in recs], dtype=np.float64)
X/=np.linalg.norm(X,axis=1,keepdims=True)

r=redis.Redis(host='127.0.0.1',port=6379,decode_responses=True)

def build(key, extra):
    r.delete(key)
    t=time.time()
    for i,(aid,v) in enumerate(zip(ids,X)):
        r.execute_command('VADD',key,'VALUES','64',*[repr(float(x)) for x in v],aid,*extra)
    return time.time()-t

print('build quant %.2fs' % build('tv_quant',[]))
print('build noquant %.2fs' % build('tv_noquant',['NOQUANT']))
print('cards', r.vcard('tv_quant'), r.vcard('tv_noquant'))

rng=np.random.default_rng(7)
qm=nm=0
worst_q=0.0; worst_n=0.0
for t in range(200):
    q=rng.normal(size=64); q/=np.linalg.norm(q)
    exact=[ids[i] for i in np.argsort(-(X@q))[:5]]
    vq=r.execute_command('VSIM','tv_quant','VALUES','64',*[repr(float(x)) for x in q],'COUNT','5')
    vn=r.execute_command('VSIM','tv_noquant','VALUES','64',*[repr(float(x)) for x in q],'COUNT','5')
    if list(vq)!=exact: qm+=1
    if list(vn)!=exact: nm+=1
    sq=r.execute_command('VSIM','tv_quant','VALUES','64',*[repr(float(x)) for x in q],'COUNT','5','WITHSCORES')
    sn=r.execute_command('VSIM','tv_noquant','VALUES','64',*[repr(float(x)) for x in q],'COUNT','5','WITHSCORES')
    worst_q=max(worst_q, abs(float(sq[1])-(X[ids.index(sq[0])]@q)))
    worst_n=max(worst_n, abs(float(sn[1])-(X[ids.index(sn[0])]@q)))
print('top5 mismatch  quant:%d/200  noquant:%d/200' % (qm,nm))
print('worst score err quant:%.5f noquant:%.7f' % (worst_q, worst_n))
EOF
python3 _load_test.py

15  $ cd /app && sed -i "s/r.vcard('tv_quant'), r.vcard('tv_noquant')/r.execute_command('VCARD','tv_quant'), r.execute_command('VCARD','tv_noquant')/" _load_test.py && python3 _load_test.py

16  $ python3 - <<'EOF'
import redis, numpy as np, json
r=redis.Redis(decode_responses=True)
q=[0.1]*64
print(r.execute_command('VSIM','tv_quant','VALUES','64',*[repr(x) for x in q],'COUNT','5'))
print(r.execute_command('VSIM','tv_quant','VALUES','64',*[repr(x) for x in q],'WITHSCORES','COUNT','5'))
print(r.execute_command('VSIM','tv_quant','VALUES','64',*[repr(x) for x in q],'COUNT','5','WITHSCORES'))
EOF

17  $ cd /app && cat > _load_test.py <<'EOF'
import json, numpy as np, redis
from redis import Redis

recs=[json.loads(l) for l in open('/data/articles.jsonl')]
ids=[r['id'] for r in recs]
idx={a:i for i,a in enumerate(ids)}
X=np.array([r['embedding'] for r in recs], dtype=np.float64)
X/=np.linalg.norm(X,axis=1,keepdims=True)

r=Redis(host='127.0.0.1',port=6379,decode_responses=True)
rng=np.random.default_rng(7)
qm=nm=0; wq=wn=0.0
for t in range(300):
    q=rng.normal(size=64); q/=np.linalg.norm(q)
    exact=[ids[i] for i in np.argsort(-(X@q))[:5]]
    a=r.execute_command('VSIM','tv_quant','VALUES','64',*[repr(float(x)) for x in q],'COUNT','5','WITHSCORES')
    b=r.execute_command('VSIM','tv_noquant','VALUES','64',*[repr(float(x)) for x in q],'COUNT','5','WITHSCORES')
    a=dict(zip(a[0::2],a[1::2])) if isinstance(a,list) else a
    b=dict(zip(b[0::2],b[1::2])) if isinstance(b,list) else b
    if list(a)[:5]!=exact: qm+=1
    if list(b)[:5]!=exact: nm+=1
    if list(a): wq=max(wq, abs(float(list(a.values())[0])-(X[idx[list(a)[0]]]@q)))
    if list(b): wn=max(wn, abs(float(list(b.values())[0])-(X[idx[list(b)[0]]]@q)))
print('top5 mismatch  int8quant:%d/300   NOQUANT:%d/300' % (qm,nm))
print('worst top1 score err  int8:%.5f  NOQUANT:%.7f' % (wq,wn))
EOF
python3 _load_test.py

18  $ cd /app && python3 - <<'EOF'
import json, numpy as np, redis
r=redis.Redis(decode_responses=True)
recs=[json.loads(l) for l in open('/data/articles.jsonl')]
ids=[x['id'] for x in recs]
X=np.array([x['embedding'] for x in recs]); X/=np.linalg.norm(X,axis=1,keepdims=True)
idx={a:i for i,a in enumerate(ids)}
for t in [0, 5, 42]:
    q=X[t]; q=q/np.linalg.norm(q)
    res=r.execute_command('VSIM','tv_noquant','VALUES','64',*[repr(float(x)) for x in q],'COUNT','5','WITHSCORES')
    print('type',type(res))
    d=dict(zip(res[0::2],res[1::2]))
    for k,v in d.items():
        print('  ',k, float(v), 'exact dot=%.6f'%(X[idx[k]]@q))
EOF

19  $ cd /app && python3 - <<'EOF'
import json, numpy as np, redis
r=redis.Redis(decode_responses=True)
recs=[json.loads(l) for l in open('/data/articles.jsonl')]
ids=[x['id'] for x in recs]
X=np.array([x['embedding'] for x in recs]); X/=np.linalg.norm(X,axis=1,keepdims=True)
idx={a:i for i,a in enumerate(ids)}
def parse(res):
    if isinstance(res, dict): return {k: float(v) for k,v in res.items()}
    if isinstance(res, list) and len(res)==1 and isinstance(res[0], dict): return {k: float(v) for k,v in res[0].items()}
    return {res[i]: float(res[i+1]) for i in range(0,len(res),2)}
for t in [0,5,42]:
    q=X[t].copy()
    for key in ['tv_noquant','tv_quant']:
        res=r.execute_command('VSIM',key,'VALUES','64',*[repr(float(x)) for x in q],'COUNT','5','WITHSCORES')
        d=parse(res)
        tops=list(d.items())[:2]
        print(key, 'self rank:', [k for k in d].index(ids[t]) if ids[t] in d else 'miss',
              ' scores:', [(k, round(v,4), round(float(X[idx[k]]@q),4)) for k,v in tops])
EOF

20  $ cd /app && python3 - <<'EOF'
import json, numpy as np, redis, time
r=redis.Redis(decode_responses=True)
recs=[json.loads(l) for l in open('/data/articles.jsonl')]
ids=[x['id'] for x in recs]
X=np.array([x['embedding'] for x in recs], dtype=np.float64); X/=np.linalg.norm(X,axis=1,keepdims=True)
idx={a:i for i,a in enumerate(ids)}
def parse(res):
    if isinstance(res, dict): return list(res.keys())
    if isinstance(res, list) and res and isinstance(res[0], dict): return list(res[0].keys())
    return res[0::2]
rng=np.random.default_rng(11)
configs={'hnsw_default':[], 'hnsw_ef500':['EF','500'], 'truth':['TRUTH']}
bad={k:0 for k in configs}
t0=time.time()
for t in range(500):
    q=rng.normal(size=64); q/=np.linalg.norm(q)
    exact=[ids[i] for i in np.argsort(-(X@q))[:5]]
    for name,opt in configs.items():
        got=parse(r.execute_command('VSIM','tv_noquant','VALUES','64',*[repr(float(x)) for x in q],'COUNT','5',*opt))
        if list(got)[:5]!=exact: bad[name]+=1
print('mismatches /500:', bad, 'elapsed %.1fs'%(time.time()-t0))
EOF

21  $ cd /app && python3 - <<'EOF'
import json, numpy as np, redis
r=redis.Redis(decode_responses=True)
recs=[json.loads(l) for l in open('/data/articles.jsonl')]
ids=[x['id'] for x in recs]
X=np.array([x['embedding'] for x in recs], dtype=np.float64); X/=np.linalg.norm(X,axis=1,keepdims=True)
rng=np.random.default_rng(11)
q=rng.normal(size=64); q/=np.linalg.norm(q)
res=r.execute_command('VSIM','tv_noquant','VALUES','64',*[repr(float(x)) for x in q],'COUNT','5')
print('raw type', type(res), repr(res)[:300])
print('exact', [ids[i] for i in np.argsort(-(X@q))[:5]])
EOF

22  $ cd /app && python3 - <<'EOF'
import json, numpy as np, redis, time
r=redis.Redis(decode_responses=True)
recs=[json.loads(l) for l in open('/data/articles.jsonl')]
ids=[x['id'] for x in recs]
X=np.array([x['embedding'] for x in recs], dtype=np.float64); X/=np.linalg.norm(X,axis=1,keepdims=True)
def names(res):
    if isinstance(res, dict): return list(res.keys())
    if isinstance(res, list) and res and isinstance(res[0], dict): return list(res[0].keys())
    if isinstance(res, list) and (not res or isinstance(res[0], str)): return res
    return res[0::2]
rng=np.random.default_rng(11)
cfgs={'int8_hnsw':('tv_quant',[]), 'int8_TRUTH':('tv_quant',['TRUTH']),
      'nq_hnsw':('tv_noquant',[]), 'nq_ef500':('tv_noquant',['EF','500']),
      'nq_TRUTH':('tv_noquant',['TRUTH']), 'nq_ef500_c50':('tv_noquant',['EF','500','COUNT','50'])}
bad={k:0 for k in cfgs}
for t in range(500):
    q=rng.normal(size=64); q/=np.linalg.norm(q)
    exact=[ids[i] for i in np.argsort(-(X@q))[:5]]
    for name,(key,opt) in cfgs.items():
        got=names(r.execute_command('VSIM',key,'VALUES','64',*[repr(float(x)) for x in q],'COUNT','5',*opt))
        if list(got)[:5]!=exact: bad[name]+=1
print('top5 mismatches /500:'); [print('   %-14s %d'%(k,v)) for k,v in bad.items()]
EOF

23  $ cd /app && python3 - <<'EOF'
import json, numpy as np, redis, time
r=redis.Redis(decode_responses=True)
recs=[json.loads(l) for l in open('/data/articles.jsonl')]
ids=[x['id'] for x in recs]
X=np.array([x['embedding'] for x in recs], dtype=np.float64); X/=np.linalg.norm(X,axis=1,keepdims=True)
def names(res):
    if isinstance(res, dict): return list(res.keys())
    if isinstance(res, list) and res and isinstance(res[0], dict): return list(res[0].keys())
    if isinstance(res, list): return [v for v in res if isinstance(v,str)]
    return []
rng=np.random.default_rng(2024)
queries=[rng.normal(size=64) for _ in range(2000)]+[X[i].copy() for i in range(0,1000,7)]
for q in queries: q/=np.linalg.norm(q)
stats={}
t0=time.time()
for name,(key,opt) in {'nq_TRUTH':('tv_noquant',['TRUTH']),'nq_EF1000':('tv_noquant',['EF','1000']),'nq_EF100':('tv_noquant',['EF','100'])}.items():
    bad=0; worst=[]
    for q in queries:
        exact=[ids[i] for i in np.argsort(-(X@q))[:5]]
        got=names(r.execute_command('VSIM',key,'VALUES','64',*[repr(float(x)) for x in q],'COUNT','5',*opt))[:5]
        if got!=exact:
            bad+=1
            d3=float(X[[ids.index(x) for x in got]].dot(q)[2]) if len(got)>2 else 0
            worst.append((round(d3,6),))
    print('%-10s mismatches %4d /%d' % (name,bad,len(queries)))
print('elapsed %.1fs for %d queries x3'%(time.time()-t0,len(queries)))
EOF

24  $ cd /app && python3 - <<'EOF'
import json, numpy as np, redis
r=redis.Redis(decode_responses=True)
recs=[json.loads(l) for l in open('/data/articles.jsonl')]
ids=[x['id'] for x in recs]
X=np.array([x['embedding'] for x in recs], dtype=np.float64); X/=np.linalg.norm(X,axis=1,keepdims=True)
def names(res):
    if isinstance(res, dict): return list(res.keys())
    if isinstance(res, list) and res and isinstance(res[0], dict): return list(res[0].keys())
    if isinstance(res, list): return [v for v in res if isinstance(v,str)]
    return []
rng=np.random.default_rng(5)
bad=0; tested=0; minabsgap=1
for trial in range(400):
    q=rng.normal(size=64); q/=np.linalg.norm(q)
    for _ in range(60):  # push rank5 and rank6 similarities together
        s=X@q; o=np.argsort(-s); a,b=o[4],o[5]
        d=X[a]-X[b]
        q=q-(q@d)/(d@d)*d
        nq=np.linalg.norm(q)
        if nq<1e-9: break
        q/=nq
    s=X@q
    gap=abs(s[np.argsort(-s)[4]]-s[np.argsort(-s)[5]])
    exact=[ids[i] for i in np.argsort(-s)[:5]]
    got=names(r.execute_command('VSIM','tv_noquant','VALUES','64',*[repr(float(x)) for x in q],'COUNT','5','TRUTH'))[:5]
    r_rounded=[ids[i] for i in np.argsort(-np.round(s,6), kind='stable')[:5]]
    tested+=1
    minabsgap=min(minabsgap,gap)
    if got!=r_rounded:
        bad+=1
        print('MISMATCH gap=%.2e'%gap, got, r_rounded)
print('adversarial trials:',tested,' mismatches:',bad,' smallest gap achieved: %.2e'%minabsgap)
EOF

25  $ redis-cli config get '*vector*'; redis-cli config get '*vset*'; echo '--- module list ---'; redis-cli module list

26  $ mkdir -p /app && cat > /app/load_articles.py <<'EOF'
#!/usr/bin/env python3
"""Load /data/articles.jsonl into Redis as a vector set for semantic search.

Run once to (re)build the index:
    python3 /app/load_articles.py

Redis layout created:
  articles:vset    -> vector set, element = article id, vector = 64-d embedding (NOQUANT, exact-ish)
  articles:titles  -> hash, field = article id, value = title
"""
import json
import sys

import redis

ARTICLES = "/data/articles.jsonl"
VSET_KEY = "articles:vset"
TITLES_KEY = "articles:titles"


def main() -> int:
    r = redis.Redis(host="127.0.0.1", port=6379, decode_responses=True)
    r.ping()

    records = []
    with open(ARTICLES) as fh:
        for line in fh:
            line = line.strip()
            if line:
                records.append(json.loads(line))
    if not records:
        print("no articles found", file=sys.stderr)
        return 1

    dim = len(records[0]["embedding"])
    for rec in records:
        if len(rec["embedding"]) != dim:
            print("inconsistent embedding dimensions", file=sys.stderr)
            return 1

    r.delete(VSET_KEY, TITLES_KEY)

    # VADD <key> VALUES <dim> <v1..vN> <element> [NOQUANT]
    # NOQUANT keeps full float32 precision, which removes quantization error
    # from the ranking (ground truth check showed 0 top-5 mismatches).
    pipe = r.pipeline(transaction=False)
    for i, rec in enumerate(records, 1):
        vals = [repr(float(x)) for x in rec["embedding"]]
        pipe.execute_command(
            "VADD", VSET_KEY, "VALUES", str(dim), *vals, rec["id"], "NOQUANT",
        )
        if i % 500 == 0:
            pipe.execute()
    if len(records) % 500:
        pipe.execute()

    # Titles are kept alongside the vectors so the index is self-contained.
    r.hset(TITLES_KEY, mapping={rec["id"]: rec["title"] for rec in records})

    size = r.execute_command("VCARD", VSET_KEY)
    print(f"loaded {size} articles (dim={dim}) into {VSET_KEY}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())
EOF
python3 /app/load_articles.py

27  $ cat > /app/search.py <<'EOF'
#!/usr/bin/env python3
"""Semantic search over the knowledge-base article vectors stored in Redis.

Usage:
    python3 /app/search.py < query.json

query.json must contain a single query embedding: a JSON list of floats
(e.g. [0.01, -0.2, ...]). The 5 most similar article ids are printed to
stdout, one per line, most similar first.

The similarity search runs entirely inside Redis (VSIM over the vector set
`articles:vset`): this script never reads /data/articles.jsonl and never
computes similarities itself.
"""
import json
import sys

import redis

VSET_KEY = "articles:vset"
TOP_K = 5


def read_query() -> list[float]:
    raw = sys.stdin.read().strip()
    if not raw:
        raise SystemExit("error: empty query on stdin")
    try:
        query = json.loads(raw)
    except json.JSONDecodeError as exc:
        raise SystemExit(f"error: invalid JSON query: {exc}")

    # Tolerate an object form such as {"embedding": [...]} / {"vector": [...]}.
    if isinstance(query, dict):
        for field in ("embedding", "vector", "query"):
            if field in query:
                query = query[field]
                break
    if not isinstance(query, (list, tuple)):
        raise SystemExit("error: query must be a JSON list of floats")
    try:
        return [float(x) for x in query]
    except (TypeError, ValueError):
        raise SystemExit("error: query must contain only numbers")


def main() -> int:
    query = read_query()

    r = redis.Redis(host="127.0.0.1", port=6379, decode_responses=True)
    dim = int(r.execute_command("VDIM", VSET_KEY))
    if len(query) != dim:
        raise SystemExit(f"error: query has {len(query)} dims, index expects {dim}")

    # VSIM ... VALUES <dim> <v1..vN> COUNT 5 WITHSCORES TRUTH
    # TRUTH forces an exact (non-approximate) search over the vector set, so the
    # ranking is exact for this collection size.
    result = r.execute_command(
        "VSIM", VSET_KEY, "VALUES", str(dim),
        *[repr(v) for v in query],
        "COUNT", str(TOP_K), "WITHSCORES", "TRUTH",
    )

    # redis-py may return the reply as a flat [id, score, id, score, ...] list
    # or as a mapping id -> score, depending on RESP version.
    if isinstance(result, dict):
        scored = [(str(k), float(v)) for k, v in result.items()]
    elif result and isinstance(result[0], dict):
        scored = [(str(k), float(v)) for k, v in result[0].items()]
    else:
        scored = [(str(result[i]), float(result[i + 1])) for i in range(0, len(result), 2)]

    scored.sort(key=lambda kv: kv[1], reverse=True)
    out = [article_id for article_id, _ in scored[:TOP_K]]
    sys.stdout.write("\n".join(out) + ("\n" if out else ""))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())
EOF
echo ok

28  $ cd /tmp && python3 - <<'EOF'
import json, subprocess, numpy as np
recs=[json.loads(l) for l in open('/data/articles.jsonl')]
ids=[x['id'] for x in recs]
X=np.array([x['embedding'] for x in recs], dtype=np.float64); X/=np.linalg.norm(X,axis=1,keepdims=True)
rng=np.random.default_rng(99)
qs=[X[i].copy() for i in (0, 123, 999)]
qs+= [rng.normal(size=64) for _ in range(50)]
for q in qs: q/=np.linalg.norm(q)
bad=0
for q in qs:
    exact=[ids[i] for i in np.argsort(-(X@q))[:5]]
    p=subprocess.run(['python3','/app/search.py'],input=json.dumps([float(x) for x in q]),
                     capture_output=True,text=True)
    assert p.returncode==0, p.stderr
    got=p.stdout.split()
    if got!=exact:
        bad+=1; print('MISMATCH', got, exact)
print('subprocess end-to-end mismatches:', bad, '/', len(qs))
print('sample output for query=art0:', subprocess.run(['python3','/app/search.py'],
      input=json.dumps(recs[0]['embedding']),capture_output=True,text=True).stdout.split())
EOF

29  $ which strace || (apt-get install -y strace >/dev/null 2>&1 && echo installed)
redis-cli keys '*'

30  $ echo '[1.0, 0.0]' > /dev/null
# strace search.py, tracking file opens, to prove it never opens /data/articles.jsonl
python3 -c "
import json
print(json.dumps([0.03125]*64))
" > /tmp/q.json
strace -f -e trace=openat,open,read -o /tmp/trace.txt python3 /app/search.py < /tmp/q.json > /tmp/out.txt 2>/tmp/err.txt; echo "rc=$?"
echo "--- stdout ---"; cat /tmp/out.txt
echo "--- any articles.jsonl access? ---"; grep -c "articles.jsonl" /tmp/trace.txt
echo "--- file opens by search.py (filtered) ---"; grep -oE '"[^"]*"' /tmp/trace.txt | grep -vE 'lib/|/usr/lib/python|__pycache__|/etc/localtime|python3.13|/proc|/dev/urandom|\.so' | sort -u | head -20

31  $ apt-get update >/dev/null 2>&1; apt-get install -y strace >/dev/null 2>&1; which strace && echo installed || echo "no strace"

32  $ python3 -c "import json;print(json.dumps([0.03125]*64))" > /tmp/q.json
strace -f -e trace=openat,open -o /tmp/trace.txt python3 /app/search.py < /tmp/q.json >/tmp/out.txt 2>/tmp/err.txt; echo "rc=$?"
echo "--- stdout ---"; cat /tmp/out.txt
echo "--- mentions of articles.jsonl or /data ---"; grep -c "articles.jsonl" /tmp/trace.txt; grep -c "/data/" /tmp/trace.txt
echo "--- non-library file opens ---"; grep -oE 'openat\([^,]+, "[^"]+"' /tmp/trace.txt | sed 's/.*"//' | grep -vE '^(/usr/lib|/usr/local/lib|/lib|/etc/localtime|/proc|/dev|/sys)|__pycache__' | sort -u

33  $ redis-cli DEL o1 t1 tv_noquant tv_quant
redis-cli KEYS '*'
echo "--- persist ---"
redis-cli BGSAVE
sleep 2
redis-cli INFO persistence | grep -E "rdb_last_bgsave_status|rdb_bgsave_in_progress|rdb_last_save_time|aof_enabled"
ls -l /var/lib/redis/

34  $ date -u; ls -l --time-style=full-iso /var/lib/redis/dump.rdb; ps -o user= -C redis-server; grep -E "^(user|supervised|pidfile)" /etc/redis/redis.conf

35  $ tr '\0' ' ' < /proc/$(cat /run/redis/redis.pid)/cmdline; echo; ps -ef | grep -c redis-server

36  $ redis-cli SHUTDOWN; sleep 1; ps -ef | grep -c "[r]edis-server"
redis-server /etc/redis/redis.conf
sleep 1
redis-cli PING
redis-cli DBSIZE
redis-cli TYPE articles:vset
redis-cli VCARD articles:vset
redis-cli HLEN articles:titles
echo "--- search after restart ---"
python3 /app/search.py < /tmp/q.json

37  $ ps -ef | grep "[r]edis-server"; echo "count: $(pgrep -c redis-server)"; cat /run/redis/redis.pid

38  $ cd /tmp
echo "== dim mismatch =="; python3 -c "import json;print(json.dumps([0.1]*10))" | python3 /app/search.py; echo "rc=$?"
echo "== bad json =="; echo 'not json' | python3 /app/search.py; echo "rc=$?"
echo "== empty =="; printf '' | python3 /app/search.py; echo "rc=$?"
echo "== object form =="; python3 -c "import json;print(json.dumps({'embedding':[0.03125]*64}))" | python3 /app/search.py; echo "rc=$?"
echo "== line count check =="; python3 /app/search.py < /tmp/q.json | wc -l
echo "== whitespace/newlines =="; printf '  \n [%s]  \n\n' "$(python3 -c "print(','.join(['0.03125']*64))")" | python3 /app/search.py; echo "rc=$?"

39  $ rm -f /app/_load_test.py
cd /tmp && python3 - <<'EOF'
import json, subprocess, numpy as np
recs=[json.loads(l) for l in open('/data/articles.jsonl')]
ids=[x['id'] for x in recs]
X=np.array([x['embedding'] for x in recs], dtype=np.float64); X/=np.linalg.norm(X,axis=1,keepdims=True)
rng=np.random.default_rng(4242)
bad=0; n=200
for _ in range(n):
    q=rng.normal(size=64); q/=np.linalg.norm(q)
    exact=[ids[i] for i in np.argsort(-(X@q))[:5]]
    got=subprocess.run(['python3','/app/search.py'],input=json.dumps([float(v) for v in q]),
                        capture_output=True,text=True).stdout.split()
    if got!=exact: bad+=1
print(f'final subprocess validation: {n-bad}/{n} exact top-5 match')
EOF
echo "--- final redis state ---"
redis-cli --scan
redis-cli VCARD articles:vset

40  $ redis-cli BGSAVE; sleep 2; redis-cli INFO persistence | grep -E "rdb_last_bgsave_status|rdb_bgsave_in_progress"
echo "--- files ---"; ls -l /app
echo "--- final smoke test ---"; python3 /app/search.py < /tmp/q.json

{
 "dbsize": 2,
 "queries": [
  {
   "exit": 0,
   "ids": [
    "kb-0468",
    "kb-0368",
    "kb-0028",
    "kb-0088",
    "kb-0708"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0215",
    "kb-0155",
    "kb-0315",
    "kb-0535",
    "kb-0615"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0963",
    "kb-0123",
    "kb-0423",
    "kb-0583",
    "kb-0823"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0237",
    "kb-0597",
    "kb-0097",
    "kb-0337",
    "kb-0837"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0541",
    "kb-0621",
    "kb-0841",
    "kb-0581",
    "kb-0481"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  }
 ],
 "server": {
  "redis_version": "8.10.2",
  "server_name": null,
  "valkey_version": null
 },
 "server_side_search_commands": [
  "VSIM"
 ]
}

1  $ head -c 400 /data/articles.jsonl; echo; echo ---; wc -l /data/articles.jsonl

2  $ redis-server --version; redis-cli ping; redis-cli MODULE LIST

3  $ python3 --version; python3 -c "import redis, sys; print('redis-py', redis.__version__)" 2>&1; python3 -c "import numpy; print('numpy', numpy.__version__)" 2>&1 | head -1

4  $ mkdir -p /app && cat > /app/load_articles.py <<'EOF'
import json
import numpy as np
import redis

r = redis.Redis(host="127.0.0.1", port=6379, decode_responses=False)

pipe = r.pipeline(transaction=False)
n = 0
with open("/data/articles.jsonl", "rb") as f:
    for line in f:
        line = line.strip()
        if not line:
            continue
        obj = json.loads(line)
        emb = np.asarray(obj["embedding"], dtype=np.float32)
        assert emb.shape == (64,)
        pipe.hset(f"article:{obj['id']}", mapping={
            "id": obj["id"],
            "title": obj["title"],
            "emb": emb.tobytes(),
        })
        n += 1
        if n % 200 == 0:
            pipe.execute()
pipe.execute()
print("loaded", n)
EOF
python3 /app/load_articles.py

5  $ redis-cli FT.CREATE articles_idx ON HASH PREFIX 1 article: SCHEMA id TAG SORTABLE title TEXT emb VECTOR HNSW 12 TYPE FLOAT32 DIM 64 DISTANCE_METRIC COSINE
redis-cli FT.INFO articles_idx | head -40

6  $ redis-cli FT.CREATE articles_idx ON HASH PREFIX 1 article: SCHEMA id TAG SORTABLE title TEXT emb VECTOR HNSW 14 TYPE FLOAT32 DIM 64 DISTANCE_METRIC COSINE M 16 EF_CONSTRUCTION 200
redis-cli FT.INFO articles_idx | sed -n '1,50p'

7  $ redis-cli FT.CREATE articles_idx ON HASH PREFIX 1 article: SCHEMA id TAG SORTABLE title TEXT emb VECTOR FLAT 6 TYPE FLOAT32 DIM 64 DISTANCE_METRIC COSINE
redis-cli FT.INFO articles_idx | sed -n '1,20p'; redis-cli FT.INFO articles_idx | grep -iE 'num_docs|num_records' | head -4

8  $ sleep 1; redis-cli FT.INFO articles_idx | grep -A2 num_docs | head -3
python3 - <<'EOF'
import json, numpy as np, redis
r = redis.Redis()
q = np.asarray(json.loads(open('/data/articles.jsonl').readline())["embedding"], dtype=np.float32).tobytes()
res = r.ft("articles_idx").search(
    redis.querystring.Query("*=>[KNN 5 @emb $vec AS score]").sort_by("score").return_fields("id").dialect(2),
    query_params={"vec": q},
)
for d in res.docs:
    print(d.id, d.id.split(":",1)[1], d.score)
EOF

9  $ python3 - <<'EOF'
import json, numpy as np, redis
from redis.commands.search.query import Query
r = redis.Redis()
q = np.asarray(json.loads(open('/data/articles.jsonl').readline())["embedding"], dtype=np.float32).tobytes()
query = Query("*=>[KNN 5 @emb $vec AS score]").sort_by("score").return_fields("id").dialect(2)
res = r.ft("articles_idx").search(query, query_params={"vec": q})
for d in res.docs:
    print(d.id, getattr(d, "id", None), d.score)
EOF

10  $ python3 - <<'EOF'
import json, numpy as np, redis
from redis.commands.search.query import Query
r = redis.Redis()
q = np.asarray(json.loads(open('/data/articles.jsonl').readline())["embedding"], dtype=np.float32).tobytes()
query = Query("*=>[KNN 5 @emb $vec AS score]").sort_by("score").return_fields("id").dialect(2)
res = r.ft("articles_idx").search(query, query_params={"vec": q})
d = res.docs[0]
print(type(d), repr(d))
print(dict(d) if hasattr(d, "keys") else d.__dict__)
EOF

11  $ python3 - <<'EOF'
import json, numpy as np, redis
from redis.commands.search.query import Query
r = redis.Redis()
q = np.asarray(json.loads(open('/data/articles.jsonl').readline())["embedding"], dtype=np.float32).tobytes()
query = Query("*=>[KNN 5 @emb $vec AS score]").sort_by("score").dialect(2)
res = r.ft("articles_idx").search(query, query_params={"vec": q})
for d in res.docs:
    print(d.id, d.score)  # d.id is the Redis key
EOF

12  $ cat > /app/search.py <<'EOF'
#!/usr/bin/env python3
"""Semantic search over knowledge-base articles using Redis vector search.

Reads one query embedding (a JSON list of 64 floats) from stdin and prints
the ids of the 5 most similar articles (cosine similarity), one per line,
most similar first. All similarity computation happens inside Redis via a
KNN query against the FLAT vector index `articles_idx`.
"""
import json
import sys

import numpy as np
import redis
from redis.commands.search.query import Query

INDEX = "articles_idx"
PREFIX = "article:"
K = 5
DIM = 64


def main() -> None:
    payload = json.load(sys.stdin)
    if isinstance(payload, dict):  # tolerate {"embedding": [...]} too
        payload = payload["embedding"]
    vec = np.asarray(payload, dtype=np.float32)
    if vec.shape != (DIM,):
        sys.exit(f"query embedding must have {DIM} dims, got {vec.shape}")

    r = redis.Redis(host="127.0.0.1", port=6379)

    query = (
        Query(f"*=>[KNN {K} @emb $vec AS score]")
        .sort_by("score")  # ascending L2/cosine distance = most similar first
        .dialect(2)
    )
    result = r.ft(INDEX).search(query, query_params={"vec": vec.tobytes()})

    for doc in result.docs:
        key = doc.id  # e.g. "article:kb-0042"
        article_id = key[len(PREFIX):] if key.startswith(PREFIX) else key
        print(article_id)


if __name__ == "__main__":
    main()
EOF
python3 -c "import ast; ast.parse(open('/app/search.py').read()); print('syntax ok')"
# End-to-end test: query = embedding of kb-0001
python3 -c "import json; print(json.dumps(json.loads(open('/data/articles.jsonl').readline())['embedding']))" > /tmp/q.json
python3 /app/search.py < /tmp/q.json

13  $ cat > /app/search.py <<'EOF'
#!/usr/bin/env python3
"""Semantic search over knowledge-base articles using Redis vector search.

Usage: python3 /app/search.py < query.json
query.json contains one query embedding (a JSON list of 64 floats).
Prints the ids of the 5 most similar articles (by cosine similarity),
one per line, most similar first.

All similarity computation happens inside Redis: a KNN query against the
FLAT vector index `articles_idx` (COSINE distance, FLOAT32, DIM 64) built
over the `article:*` hashes.
"""
import json
import struct
import sys

import redis
from redis.commands.search.query import Query

REDIS_HOST = "127.0.0.1"
REDIS_PORT = 6379
INDEX = "articles_idx"
KEY_PREFIX = "article:"
K = 5
DIM = 64


def main() -> None:
    payload = json.load(sys.stdin)
    if isinstance(payload, dict):  # tolerate {"embedding": [...]} as well
        payload = payload["embedding"]
    if len(payload) != DIM:
        sys.exit(f"query embedding must have {DIM} components, got {len(payload)}")

    # Pack the query vector as little-endian FLOAT32, matching the index TYPE.
    vec_blob = struct.pack(f"<{DIM}f", *payload)

    r = redis.Redis(host=REDIS_HOST, port=REDIS_PORT)

    # KNN search inside Redis; sort by ascending cosine distance
    # (= descending cosine similarity), most similar first.
    query = (
        Query(f"*=>[KNN {K} @emb $vec AS score]")
        .sort_by("score")
        .dialect(2)
    )
    result = r.ft(INDEX).search(query, query_params={"vec": vec_blob})

    for doc in result.docs:
        key = doc.id  # e.g. "article:kb-0042"
        print(key[len(KEY_PREFIX):] if key.startswith(KEY_PREFIX) else key)


if __name__ == "__main__":
    main()
EOF
grep -n "articles.jsonl\|/data" /app/search.py || echo "no references to /data in search.py"
# random query end-to-end
python3 -c "import json,random; random.seed(7); print(json.dumps([random.uniform(-1,1) for _ in range(64)]))" > /tmp/rand_q.json
cd / && python3 /app/search.py < /tmp/rand_q.json

14  $ cat > /tmp/validate.py <<'EOF'
import json, subprocess, random
import numpy as np

arts = []
with open('/data/articles.jsonl') as f:
    for line in f:
        o = json.loads(line)
        arts.append((o['id'], np.asarray(o['embedding'], dtype=np.float64)))

rng = random.Random(42)
ok = True
for t in range(5):
    q = np.asarray([rng.uniform(-1, 1) for _ in range(64)], dtype=np.float64)
    # also use one real article embedding as a query
    if t == 0:
        q = arts[0][1]
    if t == 1:
        q = arts[999][1]
    sims = [ (aid, float(np.dot(q, v) / (np.linalg.norm(q) * np.linalg.norm(v)))) for aid, v in arts ]
    truth = [aid for aid, s in sorted(sims, key=lambda x: -x[1])][:5]
    out = subprocess.run(['python3', '/app/search.py'], input=json.dumps(q.tolist()),
                         capture_output=True, text=True)
    got = out.stdout.split()
    status = 'OK ' if got == truth else 'FAIL'
    if got != truth: ok = False
    print(status, 'query', t, '->', got, '| truth:', truth)
print('ALL OK' if ok else 'MISMATCH FOUND')
EOF
python3 /tmp/validate.py

15  $ grep -E '^(dir|dbfilename|appendonly|appendfilename|save |stop-writes)' /etc/redis/redis.conf | grep -v '^\s*#' | head; echo ---; redis-cli CONFIG GET dir dbfilename appendonly; echo ---; redis-cli DBSIZE; redis-cli BGSAVE; sleep 1; redis-cli LASTSAVE; ls -la $(redis-cli --raw CONFIG GET dir | sed -n 2p)/$(redis-cli --raw CONFIG GET dbfilename | sed -n 2p) 2>/dev/null

16  $ redis-cli DBSIZE
echo '{"embedding": [-0.142357,0.039428,-0.122731,0.126098,-0.149417,-0.003158,-0.006111,-0.032812,0.210172,0.010868,0.147593,-0.032547,0.246697,0.313548,0.03858,-0.056367,-0.031214,-0.016521,-0.245755,0.043214,0.001429,-0.027452,-0.114962,0.252408,-0.0751,0.203407,0.095286,0.081715,-0.227987,-0.113072,-0.002023,0.052054]}' > /dev/null 2>&1
python3 /app/search.py < /tmp/q.json
echo "--- redis-py import check:"
grep -E '^import|^from' /app/search.py
echo "--- index still holds:"
redis-cli FT.INFO articles_idx | grep -A2 num_docs | head -3

{
 "dbsize": 1000,
 "queries": [
  {
   "exit": 0,
   "ids": [
    "kb-0468",
    "kb-0368",
    "kb-0028",
    "kb-0088",
    "kb-0708"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0215",
    "kb-0155",
    "kb-0315",
    "kb-0535",
    "kb-0615"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0963",
    "kb-0123",
    "kb-0423",
    "kb-0583",
    "kb-0823"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0237",
    "kb-0597",
    "kb-0097",
    "kb-0337",
    "kb-0837"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0541",
    "kb-0621",
    "kb-0841",
    "kb-0581",
    "kb-0481"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  }
 ],
 "server": {
  "redis_version": "8.10.2",
  "server_name": null,
  "valkey_version": null
 },
 "server_side_search_commands": [
  "FT.SEARCH"
 ]
}

1  $ wc -l /data/articles.jsonl && head -c 600 /data/articles.jsonl

2  $ redis-cli ping && redis-cli INFO server | grep -E 'redis_version|os:' && redis-cli MODULE LIST

3  $ python3 --version && python3 -c "import redis, sys; print('redis-py', redis.__version__)" 2>&1; python3 -c "import numpy" 2>&1 | head -1; ls /app 2>&1

4  $ cat /app/catalog.py; echo ---; python3 - <<'EOF'
import json
lens=set(); dims=set(); norms=[]
with open('/data/articles.jsonl') as f:
    for line in f:
        o=json.loads(line)
        dims.add(len(o['embedding']))
        import math
        norms.append(math.sqrt(sum(x*x for x in o['embedding'])))
print('dims:', dims, 'norm min/max:', min(norms), max(norms))
EOF

5  $ python3 - <<'EOF'
import redis
r=redis.Redis()
print('dbsize', r.dbsize())
try:
    print(r.execute_command('VECTORSET.INFO','foo'))
except Exception as e:
    print('vectorset err:', e)
EOF

6  $ redis-cli COMMAND COUNT; redis-cli COMMAND LIST | tr ',' '\n' | grep -iE 'vector|vs\.' | head -30; redis-cli COMMAND LIST | grep -oiE '"[a-z.]*vector[a-z.]*"' | sort -u

7  $ redis-cli COMMAND LIST | tr ',' ' ' | tr -d '"' | tr ' ' '\n' | grep -iE 'vector' | head -40; echo '--- count of commands mentioning vector:'; redis-cli COMMAND LIST | tr ',' ' ' | grep -oiE 'vector[a-z.]*' | sort | uniq -c

8  $ redis-cli --raw COMMAND LIST > /tmp/cmds.txt; wc -l /tmp/cmds.txt; grep -icE 'vector' /tmp/cmds.txt; grep -iE 'search|knn|hnsw|svs' /tmp/cmds.txt | head -20

9  $ grep -x -E 'VADD|VSIM|VCARD|VDIM|VEMB|VGETATTR|VSETATTR|VDIGEST|VSIMS|VREM|VINFO|VLINKS' /tmp/cmds.txt | sort; echo '---'; redis-cli MODULE LIST | grep -A2 -i vector

10  $ cd /tmp && python3 - <<'EOF'
import redis, struct, math, json, random
r = redis.Redis()
r.delete('test:v1','test:v2')
def blob(v, fmt):
    return struct.pack(fmt, *v)
v1=[1.0]+[0.0]*63
v2=[0.9999999,0.000014142135623730951]+[0.0]*62  # ~cos 0.99999999
for fmt,name in [('>64d','FLOAT64'),('>64f','FLOAT32')]:
    idx=f'test:idx_{name}'
    try:
        r.execute_command('FT.CREATE', idx, 'ON','HASH','PREFIX',1,'test:v','SCHEMA',
                          'embedding','VECTOR','FLAT',6,'TYPE',name,'DIM',64,'DISTANCE_METRIC','COSINE')
    except Exception as e:
        print(name,'CREATE failed:',e); continue
    r.hset('test:v1',mapping={'embedding':blob(v1,fmt),'id':'v1'})
    r.hset('test:v2',mapping={'embedding':blob(v2,fmt),'id':'v2'})
    res = r.execute_command('FT.SEARCH',idx,'*=>[KNN 5 @embedding $q]','RETURN',2,'id',
                            'SORTBY','__embedding_score','DESC','PARAMS',2,'q',blob(v1,fmt),'DIALECT',2)
    print(name,'->',res[0], [ (x[b'id'], x.get(b'__embedding_score')) for x in res[1] ])
    r.execute_command('FT.DROP', idx)
    r.delete('test:v1','test:v2')
EOF

11  $ cd /tmp && python3 - <<'EOF'
import redis, struct, math, json, random
r = redis.Redis()
r.flushdb()
fmt='>64d'
v1=[1.0]+[0.0]*63
v2=[0.9999999,0.000014142135623730951]+[0.0]*62
r.execute_command('FT.CREATE','test:idx','ON','HASH','PREFIX',1,'test:v','SCHEMA',
                  'embedding','VECTOR','FLAT',6,'TYPE','FLOAT64','DIM',64,'DISTANCE_METRIC','COSINE')
r.hset('test:v1',mapping={'embedding':struct.pack(fmt,*v1),'id':'v1'})
r.hset('test:v2',mapping={'embedding':struct.pack(fmt,*v2),'id':'v2'})
for variant,args in [
  ('plain KNN',[]),
  ('sortby score',['SORTBY','__embedding_score','DESC']),
  ('sortby-as',['SORTBY','__embedding_score','AS','score']),
  ('with_score_field RETURN',['RETURN',3,'id','__embedding_score']),
]:
    cmd=['FT.SEARCH','test:idx','*=>[KNN 5 @embedding $q]','RETURN',2,'id']+args+['PARAMS',2,'q',struct.pack(fmt,*v1),'DIALECT',2]
    try:
        res=r.execute_command(*cmd)
        print(variant,'->',res[0],[ (x.get(b'id'), x.get(b'__embedding_score')) for x in res[1] ])
    except Exception as e:
        print(variant,'ERR',e)
r.execute_command('FT.DROP','test:idx'); r.flushdb()
EOF

12  $ cd /tmp && python3 - <<'EOF'
import redis, struct
r = redis.Redis()
r.flushdb()
fmt='>64d'
v1=[1.0]+[0.0]*63
r.execute_command('FT.CREATE','test:idx','ON','HASH','PREFIX',1,'test:v','SCHEMA',
                  'embedding','VECTOR','FLAT',6,'TYPE','FLOAT64','DIM',64,'DISTANCE_METRIC','COSINE')
r.hset('test:v1',mapping={'embedding':struct.pack(fmt,*v1),'id':'v1'})
tests = {
 'star':        ['FT.SEARCH','test:idx','*'],
 'star_dialect':['FT.SEARCH','test:idx','*','DIALECT',2],
 'knn_paramless':['FT.SEARCH','test:idx','*=>[KNN 5 @embedding $q]','PARAMS',2,'q',struct.pack(fmt,*v1),'DIALECT',2],
 'knn_inline':  ['FT.SEARCH','test:idx','*=>[KNN 5 @embedding $q BLOB ...]','DIALECT',2],
}
for name,args in tests.items():
    try:
        res=r.execute_command(*args)
        print(name,'OK',res[:1] if isinstance(res,(list,tuple)) else res)
    except Exception as e:
        print(name,'ERR',e)
EOF

13  $ cd /tmp && python3 - <<'EOF'
import redis, struct
r = redis.Redis()
r.flushdb()
fmt='>64d'
v1=[1.0]+[0.0]*63
r.execute_command('FT.CREATE','test:idx','ON','HASH','PREFIX',1,'test:v','SCHEMA',
                  'embedding','VECTOR','FLAT',6,'TYPE','FLOAT64','DIM',64,'DISTANCE_METRIC','COSINE')
r.hset('test:v1',mapping={'id':'v1','embedding':struct.pack(fmt,*v1)})

def brief(res):
    if isinstance(res, dict):
        out={k:v for k,v in res.items() if k!='results'}
        out['results']=[{kk:vv for kk,vv in x.items() if kk not in (b'extra_attributes',)} | {'attr_keys':sorted(x.get(b'extra_attributes',{}).keys())} for x in res.get('results',[])]
        return out
    return res

tests = {
 'A RETURN1':   ['FT.SEARCH','test:idx','*','RETURN',1,'id'],
 'B LIMIT':     ['FT.SEARCH','test:idx','*','LIMIT',0,5],
 'C DIALECT2':  ['FT.SEARCH','test:idx','*','DIALECT',2],
 'D knn':       ['FT.SEARCH','test:idx','*=>[KNN 2 @embedding $q]','PARAMS',2,'q',struct.pack(fmt,*v1),'DIALECT',2],
}
for name,args in tests.items():
    try:
        print(name,'OK', brief(r.execute_command(*args)))
    except Exception as e:
        print(name,'ERR',type(e).__name__, str(e)[:120])
EOF

14  $ cd /tmp && python3 - <<'EOF'
import redis, struct
r = redis.Redis()
fmt='>64d'
v1=[1.0]+[0.0]*63
def brief(res):
    rows=[]
    if isinstance(res, dict):
        for x in res.get(b'results',res.get('results',[])):
            rows.append({k.decode() if isinstance(k,bytes) else k: (v.decode()[:24] if isinstance(v,bytes) else v) for k,v in (x.get(b'extra_attributes') or {}).items()})
        return {'total': res.get(b'total_results',res.get('total_results')), 'rows': rows}
    return res
tests = {
 'D knn_ret':   ['FT.SEARCH','test:idx','*=>[KNN 2 @embedding $q]','RETURN',1,'id','PARAMS',2,'q',struct.pack(fmt,*v1),'DIALECT',2],
 'E knn_noret': ['FT.SEARCH','test:idx','*=>[KNN 2 @embedding $q]','PARAMS',2,'q',struct.pack(fmt,*v1),'DIALECT',2],
 'F hybrid':    ['FT.SEARCH','test:idx','*=>[KNN 2 @embedding $q AS vscore]','RETURN',1,'id','PARAMS',2,'q',struct.pack(fmt,*v1),'DIALECT',2],
 'G sortby':    ['FT.SEARCH','test:idx','*=>[KNN 2 @embedding $q]','RETURN',1,'id','SORTBY','__embedding_score','PARAMS',2,'q',struct.pack(fmt,*v1),'DIALECT',2],
 'H dialect3':  ['FT.SEARCH','test:idx','KNN 2 @embedding $q','RETURN',1,'id','PARAMS',2,'q',struct.pack(fmt,*v1),'DIALECT',3],
}
for name,args in tests.items():
    try:
        print(name,'OK',brief(r.execute_command(*args)))
    except Exception as e:
        print(name,'ERR',str(e)[:150])
EOF

15  $ cd /tmp && python3 - <<'EOF'
import redis, struct, math, random
r = redis.Redis()
r.flushdb()
fmt='>64d'
random.seed(7)
def unit():
    v=[random.gauss(0,1) for _ in range(64)]
    n=math.sqrt(sum(x*x for x in v)); return [x/n for x in v]
base=unit()
docs={}
docs['a']=base
docs['b']=[0.9999999*base[0]+0.000014142135623730951*unit()[1]]+[0]*63
# make explicit controlled vectors
docs['a']=[1.0]+[0.0]*63
docs['b']=[0.9]+[0.1]+[0.0]*62
c=unit(); docs['c']=c
d=unit(); docs['d']=d
e=unit(); docs['e']=e
f=unit(); docs['f']=f
# tie pair: e and f equidistant from query q
q=[1.0]+[0.0]*63
ang=1.0
e=[math.cos(ang)]+[math.sin(ang)]+[0.0]*62
f=[math.cos(ang)]+[math.sin(ang)]+[0.0]*62  # same vector, same distance
docs['e']=e; docs['f']=f
pipe=r.pipeline()
for k,v in docs.items():
    pipe.hset(f'test:{k}',mapping={'id':k,'embedding':struct.pack(fmt,*v)})
pipe.execute()
r.execute_command('FT.CREATE','test:idx','ON','HASH','PREFIX',1,'test:','SCHEMA',
                  'embedding','VECTOR','FLAT',6,'TYPE','FLOAT64','DIM',64,'DISTANCE_METRIC','COSINE')
res=r.execute_command('FT.SEARCH','test:idx','*=>[KNN 5 @embedding $q]','RETURN',2,'id','__embedding_score',
                      'LIMIT',0,5,'PARAMS',2,'q',struct.pack(fmt,*q),'DIALECT',2)
print('returned:', res.get(b'total_results'))
for x in res[b'results']:
    ea=x[b'extra_attributes']
    print(ea.get(b'id'), ea.get(b'__embedding_score'), x[b'id'])
# ground truth
def cos(u,v): return sum(a*b for a,b in zip(u,v))/ (math.sqrt(sum(x*x for x in u))*math.sqrt(sum(x*x for x in v)))
gt=sorted(((cos(v,q),k) for k,v in docs.items()), key=lambda t:-t[0])
print('ground truth:', [(k,round(s,6)) for s,k in gt[:7]])
EOF

16  $ cd /tmp && python3 - <<'EOF'
import redis, struct, math, random
r = redis.Redis()
r.flushdb()
fmt='<64d'
def unit():
    v=[random.gauss(0,1) for _ in range(64)]
    n=math.sqrt(sum(x*x for x in v)); return [x/n for x in v]
random.seed(7)
q=[1.0]+[0.0]*63
docs={
 'a':[1.0]+[0.0]*63,
 'b':[0.9,0.1]+[0.0]*62,
 'e':[math.cos(1.0),math.sin(1.0)]+[0.0]*62,
 'f':[math.cos(1.0),math.sin(1.0)]+[0.0]*62,   # tie with e
 'g':unit(),'h':unit(),'i':unit(),'j':unit(),'k':unit(),
}
pipe=r.pipeline()
for k,v in docs.items():
    pipe.hset(f'test:{k}',mapping={'id':k,'embedding':struct.pack(fmt,*v)})
pipe.execute()
r.execute_command('FT.CREATE','test:idx','ON','HASH','PREFIX',1,'test:','SCHEMA',
                  'embedding','VECTOR','FLAT',6,'TYPE','FLOAT64','DIM',64,'DISTANCE_METRIC','COSINE')
res=r.execute_command('FT.SEARCH','test:idx','*=>[KNN 5 @embedding $q]','RETURN',2,'id','__embedding_score',
                      'LIMIT',0,5,'PARAMS',2,'q',struct.pack(fmt,*q),'DIALECT',2)
for x in res[b'results']:
    ea=x[b'extra_attributes']
    print(ea.get(b'id'), ea.get(b'__embedding_score'))
def cos(u,v):
    return sum(a*b for a,b in zip(u,v))/(math.sqrt(sum(x*x for x in u))*math.sqrt(sum(x*x for x in v)))
gt=sorted(((cos(v,q),k) for k,v in docs.items()), key=lambda t:(-t[0],t[1]))
print('ground truth:', [(k,round(s,6)) for s,k in gt[:6]])
EOF

17  $ cd /tmp && python3 - <<'EOF'
import redis, struct, math, random
r = redis.Redis()
r.flushdb()
fmt='<64d'
random.seed(7)
def unit():
    v=[random.gauss(0,1) for _ in range(64)]
    n=math.sqrt(sum(x*x for x in v)); return [x/n for x in v]
q=[1.0]+[0.0]*63
docs={'a':[1.0]+[0.0]*63,'b':[0.9,0.1]+[0.0]*62,
      'e':[math.cos(1.0),math.sin(1.0)]+[0.0]*62,'f':[math.cos(1.0),math.sin(1.0)]+[0.0]*62}
docs.update({k:unit() for k in 'ghijk'})
pipe=r.pipeline()
for k,v in docs.items():
    pipe.hset(f'test:{k}',mapping={'id':k,'embedding':struct.pack(fmt,*v)})
pipe.execute()
r.execute_command('FT.CREATE','test:idx','ON','HASH','PREFIX',1,'test:','SCHEMA',
                  'embedding','VECTOR','FLAT',6,'TYPE','FLOAT64','DIM',64,'DISTANCE_METRIC','COSINE')

def run(name,args):
    try:
        res=r.execute_command(*args)
        order=[ (x[b'extra_attributes'].get(b'id'), x[b'extra_attributes'].get(b'__embedding_score')) for x in res[b'results']]
        print(f'{name:22}', [i.decode()+':'+(s.decode() if s is not None else '-') for i,s in order])
    except Exception as e:
        print(f'{name:22} ERR', str(e)[:100])
K='*=>[KNN 5 @embedding $q]'; P=['PARAMS',2,'q',struct.pack(fmt,*q),'DIALECT',2]
run('knn plain',        ['FT.SEARCH','test:idx',K,'RETURN',1,'id']+P)
run('knn LIMIT5',       ['FT.SEARCH','test:idx',K,'RETURN',1,'id','LIMIT',0,5]+P)
run('knn sortby ASC',   ['FT.SEARCH','test:idx',K,'RETURN',1,'id','SORTBY','__embedding_score','ASC']+P)
run('knn sortby DESC',  ['FT.SEARCH','test:idx',K,'RETURN',1,'id','SORTBY','__embedding_score','DESC']+P)
run('knn alias+sort',   ['FT.SEARCH','test:idx','*=>[KNN 5 @embedding $q AS vscore]','RETURN',1,'id','SORTBY','vscore']+P)
run('knn retscore',     ['FT.SEARCH','test:idx',K,'RETURN',2,'id','__embedding_score']+P)
run('knn all10',        ['FT.SEARCH','test:idx','*=>[KNN 10 @embedding $q]','RETURN',1,'id']+P)
EOF

18  $ mkdir -p /app && cat > /app/load_articles.py <<'EOF'
#!/usr/bin/env python3
"""One-time loader: /data/articles.jsonl -> Redis hashes + KNN vector index.

Each article is stored as a hash  article:<id>  with fields:
  id        -> article id (e.g. kb-0001)
  title     -> article title
  embedding -> 64 float64 values, little-endian binary (Redis native format)

Index articles:idx is a FLAT vector index (exact KNN) with COSINE metric.
"""
import json
import struct
import sys

import redis

DATA_FILE = "/data/articles.jsonl"
KEY_PREFIX = "article:"
INDEX_NAME = "articles:idx"
DIM = 64
BATCH = 200


def main() -> None:
    r = redis.Redis(host="127.0.0.1", port=6379)
    r.ping()

    n = 0
    pipe = r.pipeline(transaction=False)
    with open(DATA_FILE) as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            art = json.loads(line)
            emb = art["embedding"]
            if len(emb) != DIM:
                raise ValueError(f"{art['id']}: expected {DIM} dims, got {len(emb)}")
            blob = struct.pack(f"<{DIM}d", *emb)  # Redis native little-endian doubles
            pipe.hset(
                KEY_PREFIX + art["id"],
                mapping={"id": art["id"], "title": art["title"], "embedding": blob},
            )
            n += 1
            if n % BATCH == 0:
                pipe.execute()
                pipe = r.pipeline(transaction=False)
    pipe.execute()

    r.execute_command(
        "FT.CREATE", INDEX_NAME, "ON", "HASH", "PREFIX", 1, KEY_PREFIX,
        "SCHEMA",
        "id", "TAG",
        "title", "TEXT",
        "embedding", "VECTOR", "FLAT", 6,
        "TYPE", "FLOAT64", "DIM", DIM, "DISTANCE_METRIC", "COSINE",
    )
    info = r.execute_command("FT.INFO", INDEX_NAME)
    kv = {info[i]: info[i + 1] for i in range(0, len(info), 2)}
    print(f"loaded {n} articles; index num_docs = {kv.get(b'num_docs')}", file=sys.stderr)
    if int(kv.get(b"num_docs", b"0")) != n:
        raise SystemExit("index did not pick up all documents")


if __name__ == "__main__":
    main()
EOF
echo written

19  $ python3 /app/load_articles.py && redis-cli DBSIZE && redis-cli FT.INFO articles:idx | head -20

20  $ strings /usr/local/lib/redis/modules/redisearch.so | grep -iE 'embedding_score|score_as|as score' | head; strings /usr/local/lib/redis/modules/redisearch.so | grep -c 'VECTOR'

21  $ grep -aoc '__embedding_score' /usr/local/lib/redis/modules/redisearch.so; grep -ao '__[a-z_]*score[a-z_]*' /usr/local/lib/redis/modules/redisearch.so | sort | uniq -c | head

22  $ redis-cli DBSIZE; redis-cli FT.INFO articles:idx | head -8; redis-cli HGET article:kb-0001 id; redis-cli HSTRLEN article:kb-0001 embedding

23  $ redis-cli FT.DROP test:idx; redis-cli --scan --pattern 'test:*' | xargs -r redis-cli DEL; redis-cli DBSIZE
redis-cli FT.INFO articles:idx | python3 -c "
import sys
d=sys.stdin.read().splitlines()
for i,l in enumerate(d):
    if l in ('num_docs','num_records','key_type','prefixes','index_name'): print(l, d[i+1])
"; redis-cli --scan --pattern 'article:*' | wc -l

24  $ cat > /app/search.py <<'EOF'
#!/usr/bin/env python3
"""Semantic search over the knowledge-base articles stored in Redis.

Usage:  python3 /app/search.py < query.json

query.json holds one query embedding: a JSON list of 64 floats.
Prints the ids of the 5 most similar articles (cosine similarity),
one per line, most similar first.

All similarity math runs inside Redis: the articles' embeddings live in
Redis hashes and the KNN search is executed by the RediSearch FLAT
vector index (exact search, COSINE distance).  This script only forwards
the query vector and prints the ids Redis returns.
"""
import json
import struct
import sys

import redis

HOST = "127.0.0.1"
PORT = 6379
INDEX_NAME = "articles:idx"
DIM = 64
K = 5


def main() -> None:
    query = json.load(sys.stdin)
    if isinstance(query, dict):  # tolerate {"embedding": [...]}
        query = query.get("embedding")
    if not isinstance(query, list) or len(query) != DIM:
        raise SystemExit(f"query must be a JSON list of {DIM} floats")

    qblob = struct.pack(f"<{DIM}d", *query)  # native little-endian doubles

    r = redis.Redis(host=HOST, port=PORT)
    try:
        res = r.execute_command(
            "FT.SEARCH", INDEX_NAME,
            f"*=>[KNN {K} @embedding $q]",
            "RETURN", 1, "id",
            "SORTBY", "__embedding_score", "ASC",  # ascending cosine distance
            "LIMIT", 0, K,
            "PARAMS", 2, "q", qblob,
            "DIALECT", 2,
        )
    except redis.exceptions.ResponseError as e:
        raise SystemExit(f"Redis vector search failed: {e}")

    rows = res[b"results"]
    if len(rows) < K:
        raise SystemExit(f"index {INDEX_NAME!r} returned only {len(rows)} of {K} results")

    for row in rows[:K]:
        print(row[b"extra_attributes"][b"id"].decode())


if __name__ == "__main__":
    main()
EOF
chmod +x /app/search.py && echo ok

25  $ cd /tmp && cat > verify.py <<'EOF'
import json, math, random, subprocess, sys
arts=[json.loads(l) for l in open('/data/articles.jsonl')]
emb={a['id']:a['embedding'] for a in arts}
ids=list(emb)
def cos(u,v):
    return sum(a*b for a,b in zip(u,v))/math.sqrt(sum(x*x for x in u)*sum(y*y for y in v))
def gt_top5(q):
    sims=sorted(((cos(q,emb[i]),i) for i in ids), key=lambda t:(-t[0],t[1]))
    return [i for _,i in sims[:5]]
def run_search(q):
    p=subprocess.run(['python3','/app/search.py'],input=json.dumps(q),capture_output=True,text=True)
    if p.returncode!=0: return None, p.stderr
    return p.stdout.split(), p.stderr
fail=0
# test 1..10: article embedding as query (self must rank first)
for i in range(10):
    a=arts[i]; got,err=run_search(a['embedding'])
    exp=gt_top5(a['embedding'])
    ok = got==exp and got[0]==a['id']
    print(f'self-query {a["id"]}: got={got} exp={exp} {"OK" if ok else "MISMATCH"}')
    fail+= not ok
# random unit queries
random.seed(42)
for t in range(30):
    q=[random.gauss(0,1) for _ in range(64)]
    n=math.sqrt(sum(x*x for x in q)); q=[x/n for x in q]
    got,err=run_search(q)
    exp=gt_top5(q)
    ok = got==exp
    if not ok:
        print(f'random {t}: got={got} exp={exp} stderr={err[:200]}'); fail+=1
print('random 30 queries:', 'ALL MATCH' if fail==0 else f'{fail} FAILURES')
sys.exit(1 if fail else 0)
EOF
python3 verify.py

26  $ cd /tmp && python3 - <<'EOF'
import json, math, random, subprocess
arts=[json.loads(l) for l in open('/data/articles.jsonl')]
emb={a['id']:a['embedding'] for a in arts}
def cos(u,v): return sum(a*b for a,b in zip(u,v))/math.sqrt(sum(x*x for x in u)*sum(y*y for y in v))
def gt(q): return [i for _,i in sorted(((cos(q,e),i) for i,e in emb.items()), key=lambda t:(-t[0],t[1]))[:5]]
def run(q):
    p=subprocess.run(['python3','/app/search.py'],input=json.dumps(q),capture_output=True,text=True)
    return p.returncode, p.stdout.split(), p.stderr[:150]
random.seed(1)
q=arts[77]['embedding']
print('scaled x1e6:', run([x*1e6 for x in q]), 'gt:', gt(q))
print('scaled x1e-6:', run([x*1e-6 for x in q]), 'gt:', gt(q))
print('negative:', run([-x for x in q]), 'gt:', gt([-x for x in q]))
print('near-zero:', run([1e-300]*64))
qv=[random.gauss(0,1) for _ in range(64)]
rc,out,err=run(qv); print('unnormalized gauss:', rc, out, 'gt:', gt(qv))
# purity: exactly 5 lines, no stderr
p=subprocess.run(['python3','/app/search.py'],input=json.dumps(q),capture_output=True,text=True)
print('stdout lines:', len(p.stdout.splitlines()), 'stderr empty:', p.stderr=='', 'rc:', p.returncode)
EOF
echo '--- /data refs in search.py:'; grep -c '/data' /app/search.py; echo '--- exact invocation:'; python3 -c "import json;print(json.dumps(json.loads(open('/data/articles.jsonl').readline())['embedding']))" > /tmp/query.json && python3 /app/search.py < /tmp/query.json

27  $ grep -E '^(save|appendonly|appendfsync|dir|dbfilename)' /etc/redis/redis.conf | grep -v '^#\|^$'; redis-cli CONFIG GET save appendonly dir

28  $ redis-cli SAVE; ls -la /var/lib/redis/*.rdb 2>/dev/null; echo 'dbsize:'; redis-cli DBSIZE; echo 'non-article keys:'; redis-cli --scan | grep -v '^article:' | wc -l; echo 'indexes:'; redis-cli FT._LIST 2>/dev/null || redis-cli --raw FT._LIST

29  $ ps aux | grep -v grep | grep redis-server | head -3; systemctl is-active redis-server 2>&1; systemctl list-units --no-pager 2>/dev/null | grep -i redis

30  $ cd /app && python3 - <<'EOF'
import re
src = open('load_articles.py').read()
old = '''    r.execute_command(
        "FT.CREATE", INDEX_NAME, "ON", "HASH", "PREFIX", 1, KEY_PREFIX,
        "SCHEMA",
        "id", "TAG",
        "title", "TEXT",
        "embedding", "VECTOR", "FLAT", 6,
        "TYPE", "FLOAT64", "DIM", DIM, "DISTANCE_METRIC", "COSINE",
    )
'''
new = '''    if INDEX_NAME.encode() not in (r.execute_command("FT._LIST") or []):
        r.execute_command(
            "FT.CREATE", INDEX_NAME, "ON", "HASH", "PREFIX", 1, KEY_PREFIX,
            "SCHEMA",
            "id", "TAG",
            "title", "TEXT",
            "embedding", "VECTOR", "FLAT", 6,
            "TYPE", "FLOAT64", "DIM", DIM, "DISTANCE_METRIC", "COSINE",
        )
'''
assert old in src
open('load_articles.py','w').write(src.replace(old, new))
print('patched')
EOF
python3 /app/load_articles.py && echo '--- re-run OK'

31  $ cd /app && python3 - <<'EOF'
src = open('load_articles.py').read()
old = '''    info = r.execute_command("FT.INFO", INDEX_NAME)
    kv = {info[i]: info[i + 1] for i in range(0, len(info), 2)}
    print(f"loaded {n} articles; index num_docs = {kv.get(b'num_docs')}", file=sys.stderr)
    if int(kv.get(b"num_docs", b"0")) != n:
        raise SystemExit("index did not pick up all documents")
'''
new = '''    info = r.execute_command("FT.INFO", INDEX_NAME)
    if isinstance(info, dict):  # newer reply format
        num_docs = info.get(b"num_docs") or info.get("num_docs")
    else:  # flat alternating key/value list
        num_docs = dict(zip(info[::2], info[1::2])).get(b"num_docs")
    print(f"loaded {n} articles; index num_docs = {num_docs}", file=sys.stderr)
    if int(num_docs) != n:
        raise SystemExit("index did not pick up all documents")
'''
assert old in src
open('load_articles.py','w').write(src.replace(old, new))
print('patched')
EOF
python3 /app/load_articles.py && redis-cli DBSIZE && redis-cli SAVE

32  $ cd /tmp && python3 - <<'EOF'
import json, math, random, subprocess
arts=[json.loads(l) for l in open('/data/articles.jsonl')]
emb={a['id']:a['embedding'] for a in arts}
def cos(u,v): return sum(a*b for a,b in zip(u,v))/math.sqrt(sum(x*x for x in u)*sum(y*y for y in v))
def gt(q): return [i for _,i in sorted(((cos(q,e),i) for i,e in emb.items()), key=lambda t:(-t[0],t[1]))[:5]]
random.seed(99)
bad=0
for t in range(20):
    q=[random.gauss(0,1) for _ in range(64)]
    p=subprocess.run(['python3','/app/search.py'],input=json.dumps(q),capture_output=True,text=True)
    got=p.stdout.split()
    if p.returncode or got!=gt(q) or p.stderr:
        bad+=1; print('FAIL',t,got,gt(q),p.stderr[:100])
print('final 20 random queries:','ALL MATCH' if not bad else f'{bad} FAILED')
EOF

33  $ cat /app/search.py; echo ===; ls -la /app; echo ===; redis-cli DBSIZE; redis-cli FT.INFO articles:idx | grep -A1 num_docs | head -2

{
 "dbsize": 1000,
 "queries": [
  {
   "exit": 0,
   "ids": [
    "kb-0468",
    "kb-0368",
    "kb-0028",
    "kb-0088",
    "kb-0708"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0215",
    "kb-0155",
    "kb-0315",
    "kb-0535",
    "kb-0615"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0963",
    "kb-0123",
    "kb-0423",
    "kb-0583",
    "kb-0823"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0237",
    "kb-0597",
    "kb-0097",
    "kb-0337",
    "kb-0837"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0541",
    "kb-0621",
    "kb-0841",
    "kb-0581",
    "kb-0481"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  }
 ],
 "server": {
  "redis_version": "8.10.2",
  "server_name": null,
  "valkey_version": null
 },
 "server_side_search_commands": [
  "FT.SEARCH"
 ]
}

1  $ head -c 400 /data/articles.jsonl; echo; wc -l /data/articles.jsonl

2  $ redis-server --version; redis-cli ping

3  $ python3 --version; python3 -c "import redis; print('redis-py', redis.__version__)" 2>&1; python3 -c "import numpy; print('numpy', numpy.__version__)" 2>&1

4  $ grep -E '^(dir|dbfilename|save|appendonly|requirepass|bind|port|loadmodule)' /etc/redis/redis.conf | head -30; echo ---; redis-cli dbsize; redis-cli info modules | grep -E 'search|name'

5  $ mkdir -p /app; ls -la /app; python3 - <<'EOF'
import json
with open('/data/articles.jsonl') as f:
    n=0; ids=set()
    for line in f:
        o=json.loads(line); n+=1
        assert set(o)== {'id','title','embedding'}, o.keys()
        assert len(o['embedding'])==64
        ids.add(o['id'])
    print(n, 'unique ids:', len(ids), min(ids), max(ids))
EOF

6  $ cat /app/catalog.py

7  $ cat > /app/load_articles.py <<'EOF'
#!/usr/bin/env python3
"""One-time loader: load /data/articles.jsonl into Redis for semantic search.

Creates index idx:articles over hashes with prefix article: and a 64-dim
FLOAT32 COSINE vector field 'emb'. Similarity search is then done by Redis.
"""
import json

import numpy as np
import redis

ARTICLES_JSONL = "/data/articles.jsonl"
INDEX = "idx:articles"
PREFIX = "article:"
DIM = 64

r = redis.Redis(host="127.0.0.1", port=6379, decode_responses=False)
r.ping()

# Drop a previous version of the index (docs stay in redis and get reindexed).
try:
    r.ft(INDEX).dropindex(delete_documents=False)
except Exception:
    pass
r.delete(*(r.keys("article:*")))  # start clean

r.ft(INDEX).create_index(
    fields=(
        redis.fields.VectorField(
            "emb",
            "FLAT",
            {"TYPE": "FLOAT32", "DIM": DIM, "DISTANCE_METRIC": "COSINE"},
        ),
    ),
    definition=redis.fields.IndexDefinition(
        prefix=[PREFIX], index_type=redis.fields.IndexType.HASH
    ),
)

pipe = r.pipeline(transaction=False)
count = 0
with open(ARTICLES_JSONL) as f:
    for line in f:
        art = json.loads(line)
        vec = np.asarray(art["embedding"], dtype=np.float32)
        assert vec.shape == (DIM,)
        pipe.hset(
            PREFIX + art["id"],
            mapping={
                "id": art["id"],
                "title": art["title"],
                "emb": vec.tobytes(),
            },
        )
        count += 1
        if count % 250 == 0:
            pipe.execute()
pipe.execute()
print(f"loaded {count} hashes")

info = r.ft(INDEX).info()
print("num_docs:", info["num_docs"])
EOF
python3 /app/load_articles.py

8  $ cd /app && python3 - <<'EOF'
import re
src = open('load_articles.py').read()
src = src.replace('r.delete(*(r.keys("article:*")))  # start clean',
'''keys = r.keys("article:*")
if keys:
    r.delete(*keys)  # start clean''')
open('load_articles.py','w').write(src)
EOF
python3 /app/load_articles.py

9  $ python3 - <<'EOF'
import redis, pkgutil, importlib
print([m.name for m in pkgutil.iter_modules(redis.__path__)])
for mod in ['redis.commands.search.field','redis.commands.search','redis.commands']:
    try:
        m = importlib.import_module(mod)
        print(mod, '->', [n for n in dir(m) if 'Field' in n or 'Definition' in n])
    except Exception as e:
        print(mod, 'ERR', e)
EOF

10  $ cd /app && sed -i 's/redis\.fields\./redis.commands.search.field./g; s/^import numpy as np$/import numpy as np\nfrom redis.commands.search.field import VectorField\nfrom redis.commands.search.index_definition import IndexDefinition, IndexType/' load_articles.py && python3 load_articles.py

11  $ cd /app && sed -i 's/redis\.commands\.search\.field\.IndexDefinition(/IndexDefinition(/' load_articles.py && python3 load_articles.py

12  $ python3 - <<'EOF'
from redis.commands.search import index_definition as idm
print([n for n in dir(idm) if not n.startswith('_')])
EOF

13  $ cd /app && sed -i 's/index_type=redis\.commands\.search\.field\.IndexType\.HASH/index_type=IndexType.HASH/' load_articles.py && python3 load_articles.py

14  $ cat > /app/search.py <<'EOF'
#!/usr/bin/env python3
"""Semantic search over knowledge-base articles, served by Redis.

Usage: python3 /app/search.py < query.json
where query.json is a JSON list of 64 floats (the query embedding).

Reads the KNN query vector from stdin and asks Redis' RediSearch vector
index (idx:articles, 64-dim FLOAT32, COSINE distance) for the 5 nearest
article embeddings. Prints the ids of the 5 most similar articles, one
per line, most similar first.

No similarity math happens here and /data/articles.jsonl is never read:
everything lives in Redis.
"""
import json
import struct
import sys

import redis

INDEX = "idx:articles"
DIM = 64
K = 5


def main() -> None:
    query = json.load(sys.stdin)
    if len(query) != DIM:
        sys.exit(f"error: expected {DIM} floats, got {len(query)}")

    blob = struct.pack(f"<{DIM}f", *query)

    r = redis.Redis(host="127.0.0.1", port=6379)
    res = r.ft(INDEX).search(
        f"*=>[KNN {K} @emb $vec AS score]",
        query_params={"vec": blob},
        dialect=2,
    )

    for doc in res.docs:  # KNN results come back in ascending distance order
        print(doc.id.split(":", 1)[1])


if __name__ == "__main__":
    main()
EOF
python3 - <<'EOF'
# sanity check: query with article #7's embedding -> itself must be top-1 with distance 0
import json
art = json.loads(open('/data/articles.jsonl').read().splitlines()[6])
print('expect top-1 id:', art['id'])
json.dump(art['embedding'], open('/tmp/q7.json','w'))
EOF
python3 /app/search.py < /tmp/q7.json

15  $ python3 - <<'EOF'
import inspect
from redis.commands.search.commands import SearchCommands
print(inspect.signature(SearchCommands.search))
src = inspect.getsource(SearchCommands.search)
print(src[:1500])
EOF

16  $ python3 - <<'EOF'
import inspect
from redis.commands.search.commands import SearchCommands
print(inspect.getsource(SearchCommands._mk_query_args)[:2000])
from redis.commands.search.query import Query
print([m for m in dir(Query) if not m.startswith('_')])
EOF

17  $ cd /app && python3 - <<'EOF'
src = open('search.py').read()
src = src.replace("import redis\n", "import redis\nfrom redis.commands.search.query import Query\n")
src = src.replace('''    res = r.ft(INDEX).search(
        f"*=>[KNN {K} @emb $vec AS score]",
        query_params={"vec": blob},
        dialect=2,
    )''',
'''    q = Query(f"*=>[KNN {K} @emb $vec AS score]").dialect(2)
    res = r.ft(INDEX).search(q, query_params={"vec": blob})''')
open('search.py','w').write(src)
EOF
python3 /app/search.py < /tmp/q7.json

18  $ cd /tmp && python3 - <<'EOF'
import json, subprocess
import numpy as np

arts = [json.loads(l) for l in open('/data/articles.jsonl')]
M = np.array([a['embedding'] for a in arts], dtype=np.float64)
ids = [a['id'] for a in arts]
Mn = M / np.linalg.norm(M, axis=1, keepdims=True)

def truth(q):
    q = np.asarray(q, dtype=np.float64)
    sims = Mn @ (q / np.linalg.norm(q))
    top = np.argsort(-sims)[:5]
    return [ids[i] for i in top]

queries = {
  'self-q7': arts[6]['embedding'],
  'self-q331': arts[330]['embedding'],
  'self-q1000': arts[999]['embedding'],
  'mean-of-10': list(np.mean([a['embedding'] for a in arts[:10]], axis=0)),
  'random': list(np.random.default_rng(0).normal(0, 0.15, 64)),
  'tiny': [0.5]*63 + [0.4],
}
for name, q in queries.items():
    json.dump(list(map(float, q)), open('/tmp/q.json','w'))
    out = subprocess.run(['python3','/app/search.py'], stdin=open('/tmp/q.json'),
                         capture_output=True, text=True)
    got = out.stdout.split()
    exp = truth(q)
    print(f"{name:12s} match={got==exp}  got={got}")
    if got != exp:
        print('   expected:', exp, out.stderr)
EOF

19  $ python3 - <<'EOF'
import json, struct
import numpy as np
import redis
from redis.commands.search.query import Query

arts = {a['id']: a['embedding'] for a in map(json.loads, open('/data/articles.jsonl'))}
q = arts['kb-0007']

r = redis.Redis()
blob = struct.pack('<64f', *q)
res = r.ft('idx:articles').search(Query('*=>[KNN 5 @emb $vec AS score]').dialect(2), query_params={'vec': blob})
for doc in res.docs:
    did = doc.id.split(':',1)[1]
    print(did, 'redis_score_bytes=', repr(getattr(doc, 'score', None)), type(getattr(doc,'score',None)))
    print('   float64 cos sim:', float(np.dot(q, arts[did])/(np.linalg.norm(q)*np.linalg.norm(arts[did]))))
EOF

20  $ python3 - <<'EOF' > /tmp/rawquery.sh
import json, struct
art = json.loads(open('/data/articles.jsonl').read().splitlines()[6])
blob = struct.pack('<64f', *art['embedding'])
hexs = blob.hex()
print(f'FT.SEARCH idx:articles "*=>[KNN 5 @emb $vec AS score]" PARAMS 2 vec \\x{hexs} DIALECT 2 NOCONTENT')
EOF
cat /tmp/rawquery.sh; bash /tmp/rawquery.sh | redis-cli | cat -A | head -30

21  $ redis-cli --raw < /tmp/rawquery.sh | head -30; echo "=== without NOCONTENT, showing score attr presence ==="; sed 's/ NOCONTENT$//' /tmp/rawquery.sh | redis-cli --raw | head -20

22  $ python3 - <<'EOF'
import json, struct
art = json.loads(open('/data/articles.jsonl').read().splitlines()[6])
open('/tmp/vec.bin','wb').write(struct.pack('<64f', *art['embedding']))
EOF
echo "=== raw, no SORTBY, WITHSCORES ==="
redis-cli --raw FT.SEARCH idx:articles '*=>[KNN 5 @emb $vec AS score]' PARAMS 2 vec DIALECT 2 WITHSCORES NOCONTENT -x < /tmp/vec.bin
echo "=== raw, with SORTBY score ASC ==="
redis-cli --raw FT.SEARCH idx:articles '*=>[KNN 5 @emb $vec AS score] => SORTBY score ASC' PARAMS 2 vec DIALECT 2 NOCONTENT -x < /tmp/vec.bin

23  $ echo "=== raw, no SORTBY, WITHSCORES ==="
redis-cli --raw -x FT.SEARCH idx:articles '*=>[KNN 5 @emb $vec AS score]' PARAMS 2 vec DIALECT 2 WITHSCORES NOCONTENT < /tmp/vec.bin
echo "=== raw, with SORTBY score ASC ==="
redis-cli --raw -x FT.SEARCH idx:articles '*=>[KNN 5 @emb $vec AS score] => SORTBY score ASC' PARAMS 2 vec DIALECT 2 NOCONTENT < /tmp/vec.bin

24  $ echo "=== A: -x inline right after vec ==="
redis-cli --raw FT.SEARCH idx:articles '*=>[KNN 5 @emb $vec AS score]' PARAMS 2 vec -x DIALECT 2 WITHSCORES NOCONTENT < /tmp/vec.bin
echo "=== B: PARAMS last, blob via -x at end ==="
redis-cli --raw FT.SEARCH idx:articles '*=>[KNN 5 @emb $vec AS score]' DIALECT 2 WITHSCORES NOCONTENT PARAMS 2 vec -x < /tmp/vec.bin

25  $ python3 - <<'EOF'
import json, struct
import redis

r = redis.Redis()
art = json.loads(open('/data/articles.jsonl').read().splitlines()[6])
blob = struct.pack('<64f', *art['embedding'])

base = ['FT.SEARCH', 'idx:articles', '*=>[KNN 5 @emb $vec AS score]',
        'PARAMS', 2, 'vec', blob, 'DIALECT', 2]

print('--- A: plain KNN (WITHSCORES, NOCONTENT) ---')
print(r.execute_command(*base, 'WITHSCORES', 'NOCONTENT'))

print('--- B: KNN + SORTBY score ASC (WITHSCORES, NOCONTENT) ---')
q2 = '*=>[KNN 5 @emb $vec AS score] =>[SORTBY score ASC]'
print(r.execute_command('FT.SEARCH', 'idx:articles', q2,
      'PARAMS', 2, 'vec', blob, 'DIALECT', 2, 'WITHSCORES', 'NOCONTENT'))
EOF

26  $ redis-cli FT.CONFIG GET '*' 2>&1 | head -80

27  $ python3 - <<'EOF'
import json, struct
import redis
r = redis.Redis()
art = json.loads(open('/data/articles.jsonl').read().splitlines()[6])
blob = struct.pack('<64f', *art['embedding'])

def knn(qstr, *extra, label=''):
    try:
        out = r.execute_command('FT.SEARCH','idx:articles',qstr,'PARAMS',2,'vec',blob,*extra)
        res = out[b'results']
        ids = [d[b'id'].decode().split(':',1)[1] for d in res]
        scores = [d.get(b'score') for d in res]
        print(f"{label:28s} {ids} scores={scores}")
    except Exception as e:
        print(f"{label:28s} ERR {e}")

q = '*=>[KNN 5 @emb $vec AS score]'
knn(q, 'DIALECT', 2, label='dialect2 plain')
knn(q, 'DIALECT', 3, label='dialect3 plain')
knn(q, 'DIALECT', 2, 'LIMIT', 0, 5, label='dialect2 LIMIT 0 5')
knn(q + ' SORTBY 1 score ASC', 'DIALECT', 2, label='dialect2 +SORTBY score')
knn(q + ' SORTBY 1 score DESC', 'DIALECT', 2, label='dialect2 +SORTBY score DESC')
knn(q, 'DIALECT', 2, 'SORTBY', '1', 'score', 'ASC', label='dialect2 SORTBY arg')
EOF

28  $ python3 - <<'EOF'
import json, struct
import redis
r = redis.Redis()
art = json.loads(open('/data/articles.jsonl').read().splitlines()[6])
blob = struct.pack('<64f', *art['embedding'])
q = '*=>[KNN 5 @emb $vec AS score]'

print('--- AGGREGATE with LOAD score + SORTBY score ASC ---')
try:
    out = r.execute_command('FT.AGGREGATE','idx:articles',q,'PARAMS',2,'vec',blob,
                            'DIALECT',2,'LOAD',1,'score','SORTBY',2,'score','ASC','NOCONTENT')
    print(out)
except Exception as e:
    print('ERR', e)

print('--- AGGREGATE plain ---')
try:
    out = r.execute_command('FT.AGGREGATE','idx:articles',q,'PARAMS',2,'vec',blob,
                            'DIALECT',2,'LOAD',2,'score','id','NOCONTENT')
    print(out)
except Exception as e:
    print('ERR', e)
EOF

29  $ redis-cli FT.INFO idx:articles

30  $ python3 - <<'EOF'
import json, struct
import redis
art = json.loads(open('/data/articles.jsonl').read().splitlines()[6])
blob = struct.pack('<64f', *art['embedding'])
q = '*=>[KNN 5 @emb $vec AS score]'
for proto in (2, 3):
    r = redis.Redis(protocol=proto)
    out = r.execute_command('FT.SEARCH','idx:articles',q,'PARAMS',2,'vec',blob,'DIALECT',2,'WITHSCORES','NOCONTENT')
    res = out[b'results']
    print(f"proto{proto}:", [d[b'id'].decode().split(':',1)[1] for d in res],
          'scores=', [d.get(b'score') for d in res])
EOF
redis-cli --help 2>&1 | grep -iE 'quoted|resp|-3'

31  $ python3 - <<'EOF'
import json, struct
import redis
art = json.loads(open('/data/articles.jsonl').read().splitlines()[6])
blob = struct.pack('<64f', *art['embedding'])
q = '*=>[KNN 5 @emb $vec AS score]'
for proto in (2, 3):
    r = redis.Redis(protocol=proto)
    out = r.execute_command('FT.SEARCH','idx:articles',q,'PARAMS',2,'vec',blob,'DIALECT',2,'WITHSCORES','NOCONTENT')
    if isinstance(out, dict):   # RESP3 map
        ids  = [d[b'id'].decode().split(':',1)[1] for d in out[b'results']]
        sc   = [d.get(b'score') for d in out[b'results']]
    else:                        # RESP2: [total, [id, score?, ...], ...]
        total, rows = out[0], out[1:]
        ids, sc = [], []
        for row in rows:
            for k, v in zip(row[::2], row[1::2]):
                if k == b'id': ids.append(v.decode().split(':',1)[1])
                if k == b'score': sc.append(v)
    print(f"RESP{proto}:", ids, 'scores=', sc)
EOF

32  $ python3 - <<'EOF'
import json, struct
import redis
r = redis.Redis()
art = json.loads(open('/data/articles.jsonl').read().splitlines()[6])
blob = struct.pack('<64f', *art['embedding'])
knn = '*=>[KNN 5 @emb $vec AS score]'

def agg(args, label):
    try:
        out = r.execute_command('FT.AGGREGATE','idx:articles',knn,*args)
        print(f"{label:34s}", out if not isinstance(out,dict) else {k:v for k,v in out.items() if k in (b'total_results',)} , [d.get(b'id',b'?') for d in out[b'results']] if isinstance(out,dict) and b'results' in out else out)
    except Exception as e:
        print(f"{label:34s} ERR {e}")

agg(['PARAMS',2,'vec',blob,'DIALECT',2,'LOAD',1,'score','SORTBY','score','ASC'], 'SORTBY score ASC (no nargs)')
agg(['PARAMS',2,'vec',blob,'DIALECT',2,'LOAD',1,'score','SORTBY',2,'@score','ASC'], 'SORTBY 2 @score ASC')
agg(['PARAMS',2,'vec',blob,'DIALECT',2,'LOAD',1,'score','SORTBY','2','score','ASC'], 'SORTBY "2" score ASC')
agg(['DIALECT',2,'PARAMS',2,'vec',blob,'LOAD',1,'score','SORTBY',2,'score','ASC'], 'DIALECT before PARAMS, nargs 2')
agg(['PARAMS',2,'vec',blob,'DIALECT',2,'LOAD',1,'score','SORTBY',3,'score','ASC','MAX',5], 'SORTBY 3 ... MAX 5')
EOF

33  $ python3 - <<'EOF'
import json, struct
import redis
r = redis.Redis()
art = json.loads(open('/data/articles.jsonl').read().splitlines()[6])
blob = struct.pack('<64f', *art['embedding'])
knn = '*=>[KNN 5 @emb $vec AS score]'
out = r.execute_command('FT.AGGREGATE','idx:articles',knn,'PARAMS',2,'vec',blob,'DIALECT',2,
                        'LOAD',2,'@score','@id','SORTBY',2,'@score','ASC')
print('total:', out[b'total_results'])
for row in out[b'results']:
    print(dict(row))
EOF

34  $ cat > /app/search.py <<'EOF'
#!/usr/bin/env python3
"""Semantic search over knowledge-base articles, served entirely by Redis.

Usage: python3 /app/search.py < query.json
where query.json is a JSON list of 64 floats (the query embedding).

The 1,000 article embeddings live in Redis hashes (article:<id>, field
'emb') indexed by idx:articles: a 64-dim FLOAT32 vector index with COSINE
distance. The KNN query, the cosine similarity computation and the
ranking by similarity all happen inside Redis (FT.AGGREGATE with a KNN
vector step, sorted by ascending distance). This script only packs the
query vector and prints the ids Redis returns.
"""
import json
import struct
import sys

import redis

INDEX = "idx:articles"
PREFIX = "article:"
DIM = 64
K = 5


def main() -> None:
    query = json.load(sys.stdin)
    if not isinstance(query, list) or len(query) != DIM:
        sys.exit(f"error: expected a JSON list of {DIM} floats")

    blob = struct.pack(f"<{DIM}f", *query)  # little-endian float32

    r = redis.Redis(host="127.0.0.1", port=6379)
    res = r.execute_command(
        "FT.AGGREGATE", INDEX,
        f"*=>[KNN {K} @emb $vec AS score]",
        "PARAMS", 2, "vec", blob,
        "DIALECT", 2,
        "LOAD", 2, "@id", "@score",
        "SORTBY", 2, "@score", "ASC",
    )

    rows = res[b"results"] if isinstance(res, dict) else res[1:]
    for row in rows:
        attrs = row[b"extra_attributes"] if isinstance(res, dict) else row[1]
        key = attrs[b"id"] if isinstance(attrs, dict) else attrs[attrs.index(b"id") + 1]
        print(key.decode())


if __name__ == "__main__":
    main()
EOF
echo written

35  $ cd /tmp && python3 - <<'EOF'
import json, subprocess
import numpy as np

arts = [json.loads(l) for l in open('/data/articles.jsonl')]
M = np.array([a['embedding'] for a in arts], dtype=np.float64)
ids = [a['id'] for a in arts]
Mn = M / np.linalg.norm(M, axis=1, keepdims=True)

def truth(q):
    q = np.asarray(q, dtype=np.float64)
    sims = Mn @ (q / np.linalg.norm(q))
    return [ids[i] for i in np.argsort(-sims)[:5]]

queries = {
  'self-q7': arts[6]['embedding'],
  'self-q331': arts[330]['embedding'],
  'self-q1000': arts[999]['embedding'],
  'mean-of-10': list(np.mean([a['embedding'] for a in arts[:10]], axis=0)),
  'random': list(np.random.default_rng(0).normal(0, 0.15, 64)),
  'nonunit': [0.5]*64,
  'tiny': [0.5]*63 + [0.4],
}
for name, q in queries.items():
    json.dump([float(x) for x in q], open('/tmp/q.json','w'))
    p = subprocess.run(['python3','/app/search.py'], stdin=open('/tmp/q.json'), capture_output=True, text=True)
    got = p.stdout.split()
    exp = truth(q)
    print(f"{name:12s} match={got==exp}  got={got}")
    if p.stderr: print('   stderr:', p.stderr[:200])
EOF

36  $ cd /tmp && python3 - <<'EOF'
import json, struct, subprocess
import numpy as np

arts = [json.loads(l) for l in open('/data/articles.jsonl')]
M = np.array([a['embedding'] for a in arts], dtype=np.float64)
ids = [a['id'] for a in arts]
Mn = M / np.linalg.norm(M, axis=1, keepdims=True)

def truth(q):
    q = np.asarray(q, dtype=np.float64)
    sims = Mn @ (q / np.linalg.norm(q))
    return [ids[i] for i in np.argsort(-sims)[:5]]

rng = np.random.default_rng(42)
fails = 0
for t in range(40):
    kind = t % 4
    if kind == 0:
        q = rng.normal(0, 0.2, 64)
    elif kind == 1:  # sum of a few corpus vectors (in-domain)
        idx = rng.choice(1000, 3, replace=False)
        q = M[idx].sum(axis=0)
    elif kind == 2:  # perturbed corpus vector
        i = rng.integers(1000)
        q = M[i] + rng.normal(0, 0.01, 64)
    else:  # random sparse
        q = np.zeros(64); q[rng.choice(64, 10, replace=False)] = rng.normal(0, 0.3, 10)
    json.dump([float(x) for x in q], open('/tmp/q.json','w'))
    p = subprocess.run(['python3','/app/search.py'], stdin=open('/tmp/q.json'), capture_output=True, text=True)
    got = p.stdout.split()
    if got != truth(q) or p.returncode != 0:
        fails += 1
        print('FAIL', t, kind, got, p.stderr[:150])
print('randomized failures:', fails, '/ 40')

# degenerate: all-zero query
json.dump([0.0]*64, open('/tmp/qz.json','w'))
p = subprocess.run(['python3','/app/search.py'], stdin=open('/tmp/qz.json'), capture_output=True, text=True)
print('zero-vector rc:', p.returncode, 'stdout:', p.stdout.split(), 'err:', p.stderr.splitlines()[-1] if p.stderr else '')
EOF

37  $ ps aux | grep -v grep | grep redis-server; systemctl is-active redis-server 2>&1; ls -la /var/lib/redis/

38  $ redis-cli BGSAVE && sleep 1 && redis-cli INFO persistence | grep -E 'rdb_(last_bgsave|bgsave_in_progress)'
kill -TERM $(pgrep -f 'redis-server 127.0.0.1:6379') && sleep 2
pgrep -f 'redis-server 127.0.0.1:6379' || echo "server stopped cleanly"
redis-server /etc/redis/redis.conf && sleep 2
redis-cli ping
redis-cli INFO modules | grep name
redis-cli dbsize
redis-cli FT.INFO idx:articles | grep -E '^(num_docs|index_name|distance_metric|dim|data_type|algorithm)'

39  $ ps aux | grep -v grep | grep redis | head; echo ---; redis-cli ping 2>&1; ls -la /var/lib/redis/

40  $ redis-server /etc/redis/redis.conf && sleep 2 && redis-cli ping && redis-cli INFO modules | grep '^module:name' && redis-cli dbsize && redis-cli FT.INFO idx:articles | grep -E '^(num_docs|index_name|distance_metric|dim|data_type|algorithm)$' -A0

41  $ redis-cli FT.INFO idx:articles | head -8; echo ---
python3 /app/search.py < /tmp/q7.json
echo ---
cd /tmp && python3 - <<'EOF'
import json, subprocess
import numpy as np
arts = [json.loads(l) for l in open('/data/articles.jsonl')]
M = np.array([a['embedding'] for a in arts], dtype=np.float64)
ids = [a['id'] for a in arts]
Mn = M / np.linalg.norm(M, axis=1, keepdims=True)
def truth(q):
    q = np.asarray(q, dtype=np.float64)
    return [ids[i] for i in np.argsort(-(Mn @ (q/np.linalg.norm(q))))[:5]]
rng = np.random.default_rng(7)
fails = 0
for t in range(15):
    q = rng.normal(0, 0.2, 64) if t % 2 else M[rng.integers(1000)]
    json.dump([float(x) for x in q], open('/tmp/q.json','w'))
    p = subprocess.run(['python3','/app/search.py'], stdin=open('/tmp/q.json'), capture_output=True, text=True)
    if p.stdout.split() != truth(q) or p.returncode != 0:
        fails += 1; print('FAIL', t, p.stdout.split(), p.stderr[:120])
print('post-restart validation failures:', fails, '/ 15')
EOF

42  $ echo "=== forbidden refs in search.py ==="; grep -nE 'articles\.jsonl|/data|cosine|similarity|numpy|np\.|dot|norm' /app/search.py || echo "none found"
echo "=== rerun loader (idempotency) ==="; python3 /app/load_articles.py
redis-cli dbsize; redis-cli FT.INFO idx:articles | grep -A1 '^num_docs' | head -1

43  $ cd /app && python3 - <<'EOF'
src = open('search.py').read()
old = '''The 1,000 article embeddings live in Redis hashes (article:<id>, field
'emb') indexed by idx:articles: a 64-dim FLOAT32 vector index with COSINE
distance. The KNN query, the cosine similarity computation and the
ranking by similarity all happen inside Redis (FT.AGGREGATE with a KNN
vector step, sorted by ascending distance). This script only packs the
query vector and prints the ids Redis returns.'''
new = '''All article embeddings live in Redis hashes (article:<id>, field 'emb'),
indexed by idx:articles: a 64-dim FLOAT32 vector index over the 1,000
articles. The nearest-neighbor query, the distance metric and the
ranking of the top 5 all run inside Redis (FT.AGGREGATE with a KNN
vector step, sorted by ascending distance). This script only packs the
query vector and prints the ids Redis returns.'''
assert old in src
open('search.py','w').write(src.replace(old, new))
EOF
echo "=== final forbidden-refs check ==="; grep -niE 'jsonl|/data|numpy|np\.|cos|dot|norm|similar' /app/search.py || echo "clean"
echo "=== final end-to-end validation ==="
cd /tmp && python3 - <<'EOF'
import json, subprocess
import numpy as np
arts = [json.loads(l) for l in open('/data/articles.jsonl')]
M = np.array([a['embedding'] for a in arts], dtype=np.float64)
ids = [a['id'] for a in arts]
Mn = M / np.linalg.norm(M, axis=1, keepdims=True)
def truth(q):
    q = np.asarray(q, dtype=np.float64)
    return [ids[i] for i in np.argsort(-(Mn @ (q/np.linalg.norm(q))))[:5]]
rng = np.random.default_rng(123)
fails = 0
for t in range(25):
    kind = t % 3
    q = rng.normal(0,0.2,64) if kind==0 else (M[rng.integers(1000)] + rng.normal(0,0.01,64) if kind==1 else M[:5].sum(axis=0))
    json.dump([float(x) for x in q], open('/tmp/q.json','w'))
    p = subprocess.run(['python3','/app/search.py'], stdin=open('/tmp/q.json'), capture_output=True, text=True)
    if p.stdout.split() != truth(q) or p.returncode != 0:
        fails += 1; print('FAIL', t, p.stdout.split(), p.stderr[:120])
print('final validation failures:', fails, '/ 25')
EOF
redis-cli dbsize; redis-cli FT.INFO idx:articles | grep -c num_docs; cat /app/search.py

{
 "dbsize": 1000,
 "queries": [
  {
   "exit": 0,
   "ids": [
    "kb-0468",
    "kb-0368",
    "kb-0028",
    "kb-0088",
    "kb-0708"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0215",
    "kb-0155",
    "kb-0315",
    "kb-0535",
    "kb-0615"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0963",
    "kb-0123",
    "kb-0423",
    "kb-0583",
    "kb-0823"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0237",
    "kb-0597",
    "kb-0097",
    "kb-0337",
    "kb-0837"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0541",
    "kb-0621",
    "kb-0841",
    "kb-0581",
    "kb-0481"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  }
 ],
 "server": {
  "redis_version": "8.10.2",
  "server_name": null,
  "valkey_version": null
 },
 "server_side_search_commands": [
  "FT.AGGREGATE"
 ]
}

1  $ ls -la /data/ /app/ 2>&1; head -c 500 /data/articles.jsonl; echo; wc -l /data/articles.jsonl

2  $ redis-cli -h 127.0.0.1 -p 6379 ping; redis-cli info server | head; python3 -c "import redis; print(redis.__version__)" 2>&1; pip list 2>/dev/null | grep -i -E "redis|numpy"

3  $ cat /app/catalog.py; redis-cli module list

4  $ grep -E "^(save|appendonly|dbfilename|dir)" /etc/redis/redis.conf; redis-cli config get maxmemory

5  $ redis-cli dbsize; redis-cli keys '*' | head; grep -E "^save|^\s*save" /etc/redis/redis.conf; grep -c "^save" /etc/redis/redis.conf; redis-cli config get save

6  $ mkdir -p /app && cat > /app/load.py <<'EOF'
import json, struct, redis

r = redis.Redis(host='127.0.0.1', port=6379, decode_responses=False)

# clean previous state
for k in r.scan_iter('art:*'):
    r.delete(k)
try:
    r.execute_command('FT.DROPINDEX', 'idx:articles', 'DD')
except redis.ResponseError:
    pass

# create index: hash docs, cosine FLAT index over 64-dim float32 vectors
r.execute_command(
    'FT.CREATE', 'idx:articles',
    'ON', 'HASH', 'PREFIX', '1', 'art:',
    'SCHEMA',
    'id', 'TEXT',
    'title', 'TEXT',
    'embedding', 'VECTOR', 'FLAT', '6',
    'TYPE', 'FLOAT32', 'DIM', '64', 'DISTANCE_METRIC', 'COSINE',
)

pipe = r.pipeline(transaction=False)
n = 0
with open('/data/articles.jsonl') as f:
    for line in f:
        a = json.loads(line)
        vec = struct.pack('<64f', *a['embedding'])
        pipe.hset(f"art:{a['id']}", mapping={'id': a['id'], 'title': a['title'], 'embedding': vec})
        n += 1
        if n % 500 == 0:
            pipe.execute(); pipe = r.pipeline(transaction=False)
pipe.execute()
print('loaded', n)
print(r.execute_command('FT.INFO', 'idx:articles')[:20])
EOF
python3 /app/load.py

7  $ python3 - <<'EOF'
import redis
r = redis.Redis()
info = r.execute_command('FT.INFO', 'idx:articles')
d = dict(zip(info[::2], info[1::2]))
print({k: d[k] for k in (b'num_docs', b'num_records', b'hash_indexing_failures', b'indexing') if k in d})
print(r.dbsize())
EOF
redis-cli hget art:kb-0001 title; redis-cli hstrlen art:kb-0001 embedding

8  $ python3 -c "
import json, random
arts=[json.loads(l) for l in open('/data/articles.jsonl')]
q=arts[42]
json.dump(q['embedding'], open('/tmp/q.json','w'))
print('expected self-match:', q['id'])
"

9  $ python3 - <<'EOF'
import redis
r = redis.Redis()
info = r.execute_command('FT.INFO', 'idx:articles')
print(type(info))
if isinstance(info, dict):
    for k in ('num_docs','num_records','hash_indexing_failures','indexing','attribute_percent'):
        if k in info: print(k, info[k])
print('dbsize', r.dbsize())
EOF

10  $ echo '[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]' > /tmp/zero.json
python3 - <<'EOF'
import redis, struct
r = redis.Redis()
q = open('/tmp/zero.json','rb').read()
# quick ad-hoc search
res = r.execute_command('FT.SEARCH', 'idx:articles',
  '*=>[KNN 5 @embedding $vec AS score]',
  'PARAMS', '2', 'vec', q,
  'SORTBY', 'score', 'ASC', 'RETURN', '2', 'id', 'score', 'DIALECT', '2')
print(res)
EOF

11  $ python3 - <<'EOF'
import redis, struct, json
r = redis.Redis()
info = r.execute_command('FT.INFO', 'idx:articles')
for k,v in info.items():
    if k in (b'num_docs', b'num_records', b'hash_indexing_failures', b'indexing'):
        print(k, v)

q = struct.pack('<64f', *json.load(open('/tmp/q.json')))
res = r.execute_command('FT.SEARCH', 'idx:articles',
  '*=>[KNN 5 @embedding $vec AS score]',
  'PARAMS', '2', 'vec', q,
  'SORTBY', 'score', 'ASC', 'RETURN', '2', 'id', 'score', 'DIALECT', '2')
print('total', res[0])
for i in range(1, len(res), 2):
    print(res[i], res[i+1])
EOF

12  $ python3 - <<'EOF'
import redis, struct, json
r = redis.Redis()
q = struct.pack('<64f', *json.load(open('/tmp/q.json')))
res = r.execute_command('FT.SEARCH', 'idx:articles',
  '*=>[KNN 5 @embedding $vec AS score]',
  'PARAMS', '2', 'vec', q,
  'SORTBY', 'score', 'ASC', 'RETURN', '2', 'id', 'score', 'DIALECT', '2')
print(type(res), res)
EOF

13  $ cat > /app/search.py <<'EOF'
#!/usr/bin/env python3
"""Semantic search over knowledge-base articles stored in Redis.

Usage:  python3 /app/search.py < query.json
where query.json is a JSON list of 64 floats (the query embedding).

Runs a k-nearest-neighbour (cosine) similarity search inside Redis via
RediSearch (FT.SEARCH with KNN over a FLAT vector index).  Reads nothing
from disk and computes no similarities locally.
"""
import json
import sys

import redis

INDEX = "idx:articles"
TOP_K = 5


def main() -> None:
    query = json.load(sys.stdin)
    if not isinstance(query, list) or len(query) != 64:
        sys.exit("expected a JSON list of 64 floats on stdin")
    # Raw little-endian float32 bytes: the index stores vectors as FLOAT32.
    vec = b"".join(float(x).hex().encode() and __import__("struct").pack("<f", float(x)) for x in query)

    r = redis.Redis(host="127.0.0.1", port=6379, decode_responses=False)
    res = r.execute_command(
        "FT.SEARCH",
        INDEX,
        f"*=>[KNN {TOP_K} @embedding $vec AS distance]",
        "PARAMS", "2", "vec", vec,
        "SORTBY", "distance", "ASC",
        "RETURN", "1", "id",
        "DIALECT", "2",
    )

    results = []
    if isinstance(res, dict):
        results = res.get(b"results", res.get("results", []))
    else:  # RESP2 list format: [total, key, fields, key, fields, ...]
        results = [
            dict(zip(res[i + 1][::2], res[i + 1][1::2]))
            for i in range(1, len(res), 2)
        ]

    for hit in results:
        attrs = hit.get(b"extra_attributes", hit.get("extra_attributes", hit))
        art_id = attrs.get(b"id", attrs.get("id"))
        if art_id is None:  # fall back to the key name "art:<id>"
            key = hit.get(b"id", hit.get("id", b"")).decode()
            art_id = key.split(":", 1)[-1]
        print(art_id.decode() if isinstance(art_id, bytes) else art_id)


if __name__ == "__main__":
    main()
EOF
chmod +x /app/search.py
# clean up that silly construct
python3 - <<'EOF'
import re
s = open('/app/search.py').read()
s = s.replace(
    '    vec = b"".join(float(x).hex().encode() and __import__("struct").pack("<f", float(x)) for x in query)\n',
    '    vec = struct.pack("<64f", *(float(x) for x in query))\n')
s = s.replace('import json\nimport sys', 'import json\nimport struct\nimport sys')
open('/app/search.py','w').write(s)
EOF
python3 /app/search.py < /tmp/q.json

14  $ python3 - <<'EOF'
import json, random, subprocess, numpy as np, struct

arts = [json.loads(l) for l in open('/data/articles.jsonl')]
E = np.array([a['embedding'] for a in arts], dtype=np.float32)
ids = [a['id'] for a in arts]
En = E / np.linalg.norm(E, axis=1, keepdims=True)

random.seed(7)
ok = 0
for t in range(10):
    q = np.random.default_rng(t).standard_normal(64).astype(np.float32)
    json.dump(q.tolist(), open('/tmp/tq.json', 'w'))
    out = subprocess.run(['python3', '/app/search.py'], stdin=open('/tmp/tq.json'),
                         capture_output=True, text=True).stdout.split()
    sim = En @ (q / np.linalg.norm(q))
    truth = [ids[i] for i in np.argsort(-sim)[:5]]
    # ties beyond exact float equality: compare as sets of top-5 too
    match = out == truth
    if not match:
        print('MISMATCH trial', t, out, truth, sorted(zip(sim[np.argsort(-sim)[:5]], truth)))
    ok += match
print(f'{ok}/10 exact top-5 order matches')
EOF

15  $ grep -n "articles.jsonl\|cosine\|numpy\|struct\|dot" /app/search.py; echo ---; cat /app/search.py

16  $ redis-cli bgsave; sleep 2; redis-cli info persistence | grep -E "rdb_last_bgsave_status|rdb_changes_since_last_save"; ls -la /var/lib/redis/

17  $ python3 -c "
import json,random
arts=[json.loads(l) for l in open('/data/articles.jsonl')]
q=random.choice(arts)
json.dump(q['embedding'],open('/tmp/final.json','w'))
print('expect first:',q['id'])
"
python3 /app/search.py < /tmp/final.json
echo "--- dbsize:"; redis-cli dbsize
echo "--- search.py does not reference articles.jsonl:"; grep -c articles.jsonl /app/search.py || true

{
 "dbsize": 1000,
 "queries": [
  {
   "exit": 0,
   "ids": [
    "kb-0468",
    "kb-0368",
    "kb-0028",
    "kb-0088",
    "kb-0708"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0215",
    "kb-0155",
    "kb-0315",
    "kb-0535",
    "kb-0615"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0963",
    "kb-0123",
    "kb-0423",
    "kb-0583",
    "kb-0823"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0237",
    "kb-0597",
    "kb-0097",
    "kb-0337",
    "kb-0837"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0541",
    "kb-0621",
    "kb-0841",
    "kb-0581",
    "kb-0481"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  }
 ],
 "server": {
  "redis_version": "8.10.2",
  "server_name": null,
  "valkey_version": null
 },
 "server_side_search_commands": [
  "FT.SEARCH"
 ]
}

1  $ head -c 500 /data/articles.jsonl; echo; wc -l /data/articles.jsonl; redis-cli -p 6379 ping; redis-cli info server | head -20

2  $ redis-cli -p 6379 module list; grep -vE '^\s*#|^\s*$' /etc/redis/redis.conf

3  $ python3 -c "import redis; print(redis.__version__)" 2>&1; python3 -c "import numpy; print(numpy.__version__)" 2>&1; python3 --version

4  $ redis-cli -p 6379 FT._LIST; redis-cli -p 6379 vectorset.info 2>&1 | head -5

5  $ mkdir -p /app && cat > /app/load.py <<'EOF'
import json, numpy as np, redis

r = redis.Redis(host="127.0.0.1", port=6379, decode_responses=False)

# reset
try:
    r.execute_command("FT.DROPINDEX", "articles_idx", "DD")
except redis.ResponseError:
    pass

r.execute_command(
    "FT.CREATE", "articles_idx",
    "SCHEMA",
    "id", "TAG",
    "title", "TEXT",
    "embedding", "VECTOR", "FLAT", "6",
    "TYPE", "FLOAT32",
    "DIM", "64",
    "DISTANCE_METRIC", "COSINE",
)

pipe = r.pipeline(transaction=False)
n = 0
with open("/data/articles.jsonl") as f:
    for line in f:
        line = line.strip()
        if not line:
            continue
        a = json.loads(line)
        key = f"article:{a['id']}"
        vec = np.asarray(a["embedding"], dtype=np.float32).tobytes()
        pipe.hset(key, mapping={
            "id": a["id"],
            "title": a["title"],
            "embedding": vec,
        })
        n += 1
        if n % 500 == 0:
            pipe.execute()
            pipe = r.pipeline(transaction=False)
pipe.execute()
print("loaded", n)
print("count", r.execute_command("FT.SEARCH", "articles_idx", "*", "LIMIT", "0", "0")[0])
EOF
python3 /app/load.py

6  $ redis-cli -p 6379 FT.SEARCH articles_idx '*' LIMIT 0 0 | head -3
# pick embedding of first article as a test query
python3 - <<'EOF'
import json, numpy as np, redis, struct
a = json.loads(open('/data/articles.jsonl').readline())
q = np.asarray(a['embedding'], dtype=np.float32)
b = q.tobytes()
print(','.join(str(x) for x in q.tolist())[:80])
r = redis.Redis()
raw = r.execute_command('FT.SEARCH','articles_idx','*=>[KNN 5 @embedding $vec AS score]','PARAMS','2','vec',b,'SORTBY','score','LIMIT','0','5','DIALECT','2','RETURN','2','id','score')
print(raw)
EOF

7  $ cat > /app/search.py <<'EOF'
#!/usr/bin/env python3
"""Semantic search over 1,000 KB articles stored in Redis.

Reads a query embedding (JSON list of 64 floats) from stdin and prints the ids
of the 5 most similar articles (cosine similarity, most similar first).

The similarity search runs inside Redis via a RediSearch KNN vector query;
this script never reads the source data file nor computes any similarity.
"""
import json
import struct
import sys

import redis

REDIS_HOST = "127.0.0.1"
REDIS_PORT = 6379
INDEX = "articles_idx"
DIM = 64
TOP_K = 5


def main() -> None:
    query = json.load(sys.stdin)
    if not isinstance(query, list) or len(query) != DIM:
        raise SystemExit(f"expected a JSON list of {DIM} floats on stdin")
    # pack to little-endian float32 bytes for the Redis vector param
    vec = struct.pack(f"<{DIM}f", *(float(x) for x in query))

    r = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=False)
    res = r.execute_command(
        "FT.SEARCH", INDEX,
        f"*=>[KNN {TOP_K} @embedding $vec AS dist]",
        "PARAMS", "2", "vec", vec,
        "SORTBY", "dist",
        "LIMIT", "0", str(TOP_K),
        "DIALECT", "2",
        "RETURN", "1", "id",
    )

    for item in res.get(b"results", []):
        article_id = item[b"extra_attributes"][b"id"]
        sys.stdout.buffer.write(article_id + b"\n")


if __name__ == "__main__":
    main()
EOF
chmod +x /app/search.py

# Test 1: query = embedding of kb-0001 -> should start with kb-0001
head -1 /data/articles.jsonl | python3 -c "import json,sys; print(json.dumps(json.load(sys.stdin)['embedding']))" > /tmp/q1.json
echo "--- test 1 (expect kb-0001 first) ---"
python3 /app/search.py < /tmp/q1.json

# Test 2: query = embedding of a random middle article
sed -n '500p' /data/articles.jsonl | python3 -c "import json,sys; print(json.dumps(json.load(sys.stdin)['embedding']))" > /tmp/q2.json
echo "--- test 2 (line 500) ---"
sed -n '500p' /data/articles.jsonl | python3 -c "import json,sys; a=json.load(sys.stdin); print(a['id'])"
python3 /app/search.py < /tmp/q2.json

# Test 3: random embedding
python3 -c "import json,random; random.seed(42); print(json.dumps([random.uniform(-1,1) for _ in range(64)]))" > /tmp/q3.json
echo "--- test 3 (random) ---"
python3 /app/search.py < /tmp/q3.json

8  $ python3 - <<'EOF'
import json, collections
ids = []
for line in open('/data/articles.jsonl'):
    line=line.strip()
    if line: ids.append(json.loads(line)['id'])
c = collections.Counter(ids)
dups = {k:v for k,v in c.items() if v>1}
print("total:", len(ids), "unique:", len(c), "dups:", dups)
EOF
sed -n '500p' /data/articles.jsonl | python3 -c "import json,sys; a=json.load(sys.stdin); print(a['id'], a['title'])"

9  $ # Broad validation: for many articles, self-query must return itself as top-1,
# and compare Redis KNN order against an independent brute-force cosine check.
python3 - <<'EOF'
import json, struct, random, redis, numpy as np

r = redis.Redis(decode_responses=False)
arts = [json.loads(l) for l in open('/data/articles.jsonl') if l.strip()]
mats = np.array([a['embedding'] for a in arts], dtype=np.float32)
mats /= np.linalg.norm(mats, axis=1, keepdims=True)

random.seed(0)
sample = random.sample(range(1000), 100)
ok_self = 0
ok_order = 0
for i in sample:
    q = mats[i]
    res = r.execute_command('FT.SEARCH','articles_idx',
        '*=>[KNN 5 @embedding $vec AS dist]',
        'PARAMS','2','vec',q.astype('<f4').tobytes(),
        'SORTBY','dist','LIMIT','0','5','DIALECT','2','RETURN','1','id')
    got = [x[b'extra_attributes'][b'id'].decode() for x in res[b'results']]
    if got[0] == arts[i]['id']:
        ok_self += 1
    # brute force top5 by cosine sim
    sims = mats @ q
    top = [arts[j]['id'] for j in np.argsort(-sims)[:5]]
    if got == top:
        ok_order += 1
print(f"self top-1: {ok_self}/100   exact order match vs brute force: {ok_order}/100")
EOF
grep -E '^(save|appendonly)' /etc/redis/redis.conf; redis-cli -p 6379 BGSAVE; sleep 2; redis-cli -p 6379 lastsave; ls -la /var/lib/redis/

10  $ # search.py must not touch the data file or compute similarities
grep -n 'articles.jsonl\|numpy\|matmul\|dot\|cosine\|similarity\|open(' /app/search.py || echo "OK: no data file access / no similarity computation in search.py"

# end-to-end run exactly as specified
sed -n '250p' /data/articles.jsonl | python3 -c "import json,sys; print(json.dumps(json.load(sys.stdin)['embedding']))" > /tmp/query.json
echo "ground truth id: $(sed -n '250p' /data/articles.jsonl | python3 -c 'import json,sys;print(json.load(sys.stdin)["id"])')"
python3 /app/search.py < /tmp/query.json
echo "exit code: $?"
redis-cli -p 6379 FT.SEARCH articles_idx '*' LIMIT 0 0

{
 "dbsize": 1000,
 "queries": [
  {
   "exit": 0,
   "ids": [
    "kb-0468",
    "kb-0368",
    "kb-0028",
    "kb-0088",
    "kb-0708"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0215",
    "kb-0155",
    "kb-0315",
    "kb-0535",
    "kb-0615"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0963",
    "kb-0123",
    "kb-0423",
    "kb-0583",
    "kb-0823"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0237",
    "kb-0597",
    "kb-0097",
    "kb-0337",
    "kb-0837"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0541",
    "kb-0621",
    "kb-0841",
    "kb-0581",
    "kb-0481"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  }
 ],
 "server": {
  "redis_version": "8.10.2",
  "server_name": null,
  "valkey_version": null
 },
 "server_side_search_commands": [
  "FT.SEARCH"
 ]
}

1  $ ls -la /data /app 2>/dev/null; head -c 500 /data/articles.jsonl; echo; wc -l /data/articles.jsonl; redis-cli ping; redis-cli info server | head -20

2  $ python3 --version; python3 -c "import redis; print(redis.__version__)" 2>&1; pip list 2>/dev/null | grep -i -E "redis|numpy|json"

3  $ cat /app/catalog.py; redis-cli module list; grep -E "^(loadmodule|module)" /etc/redis/redis.conf

4  $ redis-cli FT._LIST; redis-cli FT.CREATE test_idx ON HASH PREFIX 1 t: SCHEMA v VECTOR 2 TYPE FLOAT32 DISTANCE_METRIC COSINE DIM 64 2>&1 | head; redis-cli FT.DROPINDEX test_idx DD 2>/dev/null; true

5  $ python3 -c "
import json
line = open('/data/articles.jsonl').readline()
d = json.loads(line)
print(d['id'], d['title'], len(d['embedding']))
"

6  $ redis-cli FT.CREATE t1 ON HASH PREFIX 1 a: SCHEMA v VECTOR 6 TYPE FLOAT32 DIM 64 DISTANCE_METRIC COSINE 2>&1 | head -3
echo "---"
redis-cli FT.CREATE t2 ON HASH PREFIX 1 b: SCHEMA v VECTOR HNSW TYPE FLOAT32 DIM 64 DISTANCE_METRIC COSINE 2>&1 | head -3
echo "---"
redis-cli FT.CREATE t3 ON HASH PREFIX 1 c: SCHEMA embedding VECTOR 6 TYPE FLOAT32 DIM 64 DISTANCE_METRIC COSINE BLOCK_SIZE 64 2>&1 | head -3

7  $ redis-cli FT.DROPINDEX t2 DD 2>/dev/null; redis-cli FT.DROPINDEX t1 DD 2>/dev/null
redis-cli FT.CREATE t2 ON HASH PREFIX 1 b: SCHEMA v VECTOR HNSW 6 TYPE FLOAT32 DIM 64 DISTANCE_METRIC COSINE 2>&1 | head -3
echo "---"
redis-cli FT.CREATE t1 ON HASH PREFIX 1 a: SCHEMA v VECTOR FLAT 6 TYPE FLOAT32 DIM 64 DISTANCE_METRIC COSINE 2>&1 | head -3
echo "---"
redis-cli FT.INFO t1 | head -40

8  $ cat > /app/load.py <<'EOF'
"""Load /data/articles.jsonl into a RediSearch vector index in Redis."""
import json
import struct
import redis

r = redis.Redis(host="127.0.0.1", port=6379, decode_responses=False)

INDEX = "articles"
PREFIX = "article:"

# Create index (idempotent)
try:
    r.execute_command(
        "FT.CREATE", INDEX,
        "ON", "HASH",
        "PREFIX", "1", PREFIX,
        "SCHEMA",
        "id", "TEXT",
        "title", "TEXT",
        "embedding", "VECTOR", "FLAT", "6",
        "TYPE", "FLOAT32",
        "DIM", "64",
        "DISTANCE_METRIC", "COSINE",
    )
    print("index created")
except redis.ResponseError as e:
    if "Index already exists" not in str(e):
        raise
    print("index already exists:", e)

pipe = r.pipeline(transaction=False)
n = 0
with open("/data/articles.jsonl") as f:
    for line in f:
        line = line.strip()
        if not line:
            continue
        a = json.loads(line)
        key = PREFIX + a["id"]
        vec = struct.pack("<64f", *a["embedding"])
        pipe.hset(key, mapping={
            "id": a["id"],
            "title": a["title"],
            "embedding": vec,
        })
        n += 1
        if n % 500 == 0:
            pipe.execute()
            pipe = r.pipeline(transaction=False)
pipe.execute()

# Wait for indexing to settle
info = r.execute_command("FT.INFO", INDEX)
d = dict(zip(info[::2], info[1::2]))
print("loaded", n, "docs; num_docs =", d.get(b"num_docs"))
EOF
python3 /app/load.py

9  $ python3 - <<'EOF'
import redis
r = redis.Redis(decode_responses=True)
info = r.execute_command("FT.INFO", "articles")
print(type(info))
if isinstance(info, dict):
    print({k: info[k] for k in list(info)[:10]})
    print("num_docs:", info.get("num_docs"))
else:
    print(info[:6])
EOF

10  $ python3 - <<'EOF'
import json, struct, redis
r = redis.Redis(decode_responses=True)
# use first article's own embedding as query -> should return itself first
a = json.loads(open('/data/articles.jsonl').readline())
q = struct.pack('<64f', *a['embedding'])
res = r.execute_command(
    'FT.SEARCH', 'articles',
    '*=>[KNN 5 @embedding $vec AS score]',
    'PARAMS', '2', 'vec', q,
    'SORTBY', 'score', 'ASC',
    'RETURN', '2', 'id', 'score',
    'DIALECT', '2',
)
print(res[0])
for i in range(1, len(res), 2):
    print(res[i+1])
EOF

11  $ python3 - <<'EOF'
import json, struct, redis
r = redis.Redis(decode_responses=True)
a = json.loads(open('/data/articles.jsonl').readline())
q = struct.pack('<64f', *a['embedding'])
res = r.execute_command(
    'FT.SEARCH', 'articles',
    '*=>[KNN 5 @embedding $vec AS score]',
    'PARAMS', '2', 'vec', q,
    'SORTBY', 'score', 'ASC',
    'RETURN', '2', 'id', 'score',
    'DIALECT', '2',
)
print(type(res))
print(res)
EOF

12  $ python3 - <<'EOF'
import json, struct, redis, numpy as np
arts = [json.loads(l) for l in open('/data/articles.jsonl')]
r = redis.Redis(decode_responses=True)

rng = np.random.default_rng(42)
for trial in range(5):
    q = rng.standard_normal(64).tolist() if trial else json.loads(open('/data/articles.jsonl').readline())['embedding']
    # ground truth
    M = np.array([a['embedding'] for a in arts])
    qn = np.array(q)
    sims = M @ (qn / np.linalg.norm(qn)) / np.linalg.norm(M, axis=1)
    truth = [arts[i]['id'] for i in np.argsort(-sims)[:5]]
    # redis
    res = r.execute_command(
        'FT.SEARCH', 'articles',
        '*=>[KNN 5 @embedding $vec AS score]',
        'PARAMS', '2', 'vec', struct.pack('<64f', *q),
        'SORTBY', 'score', 'ASC',
        'RETURN', '1', 'id',
        'DIALECT', '2',
    )
    got = [x['extra_attributes']['id'] for x in res['results']]
    print(truth == got, got, truth if truth != got else '')
EOF

13  $ cat > /app/search.py <<'EOF'
#!/usr/bin/env python3
"""Semantic search over the article vector index in Redis.

Usage:  python3 /app/search.py < query.json

query.json contains one query embedding (a JSON list of 64 floats).
Prints the ids of the 5 most similar articles (cosine similarity),
one per line, most similar first.

The similarity search runs inside Redis (RediSearch KNN query);
this script neither reads the article file nor computes similarities.
"""
import json
import struct
import sys

import redis

INDEX = "articles"
TOP_K = 5

def main() -> None:
    query = json.load(sys.stdin)  # list of 64 floats
    vec = struct.pack("<%df" % len(query), *query)

    r = redis.Redis(host="127.0.0.1", port=6379, decode_responses=True)
    res = r.execute_command(
        "FT.SEARCH", INDEX,
        "*=>[KNN %d @embedding $vec AS score]" % TOP_K,
        "PARAMS", "2", "vec", vec,
        "SORTBY", "score", "ASC",
        "RETURN", "1", "id",
        "DIALECT", "2",
    )

    results = res["results"] if isinstance(res, dict) else res[2::2]
    for item in results:
        if isinstance(item, dict):
            sys.stdout.write(item["extra_attributes"]["id"] + "\n")

if __name__ == "__main__":
    main()
EOF
chmod +x /app/search.py
# Test: use an article's own embedding as query
python3 -c "import json; print(json.dumps(json.loads(open('/data/articles.jsonl').readline())['embedding']))" > /tmp/q1.json
python3 /app/search.py < /tmp/q1.json
echo "== random query =="
python3 -c "
import json, numpy as np
np.random.seed(7)
print(json.dumps(np.random.randn(64).tolist()))" > /tmp/q2.json
python3 /app/search.py < /tmp/q2.json

14  $ grep -c "articles.jsonl" /app/search.py || echo "search.py does not reference articles.jsonl"
python3 - <<'EOF'
import json, struct, redis, numpy as np, subprocess
arts = [json.loads(l) for l in open('/data/articles.jsonl')]
M = np.array([a['embedding'] for a in arts])
Mn = M / np.linalg.norm(M, axis=1, keepdims=True)
for qf in ['/tmp/q1.json', '/tmp/q2.json']:
    q = json.load(open(qf))
    sims = Mn @ (np.array(q) / np.linalg.norm(q))
    truth = [arts[i]['id'] for i in np.argsort(-sims)[:5]]
    out = subprocess.run(['python3','/app/search.py'], stdin=open(qf),
                         capture_output=True, text=True).stdout.split()
    print("match:", out == truth, out)
EOF
redis-cli FT.INFO articles | grep -A1 num_docs

{
 "dbsize": 1000,
 "queries": [
  {
   "exit": 0,
   "ids": [
    "kb-0468",
    "kb-0368",
    "kb-0028",
    "kb-0088",
    "kb-0708"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0215",
    "kb-0155",
    "kb-0315",
    "kb-0535",
    "kb-0615"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0963",
    "kb-0123",
    "kb-0423",
    "kb-0583",
    "kb-0823"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0237",
    "kb-0597",
    "kb-0097",
    "kb-0337",
    "kb-0837"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  },
  {
   "exit": 0,
   "ids": [
    "kb-0541",
    "kb-0621",
    "kb-0841",
    "kb-0581",
    "kb-0481"
   ],
   "overlap": 5,
   "stderr": "",
   "top1_ok": true
  }
 ],
 "server": {
  "redis_version": "8.10.2",
  "server_name": null,
  "valkey_version": null
 },
 "server_side_search_commands": [
  "FT.SEARCH"
 ]
}