learning/deep-probes/prefix_probe.py

import importlib.util, json
from pathlib import Path
root=Path(__file__).resolve().parents[2]
spec=importlib.util.spec_from_file_location("spine_prefix",root/"guest/prefix.py")
module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module)
started=[]
def prepare(request):
    started.append(json.loads(request));return len(started)
p=module.Prefix({},prepare)
chunks=['a = tool("lookup", key="bo','ok")\n','b = tool("lookup", key="shipping")\n','result = a + b']
observed=[]
for chunk in chunks:
    p.feed(chunk)
    observed.append({"chunk":chunk,"seen":p.seen,"ready":len(p.ready),"starts":len(started)})
book=json.dumps({"tool":"lookup","args":{"key":"book"}})
shipping=json.dumps({"tool":"lookup","args":{"key":"shipping"}})
claims=[p.claim(shipping),p.claim(book),p.claim(shipping),p.claim(shipping)]
assert claims == [None,1,2,None]
compile(p.source,"<final>","exec")
q=module.Prefix({},lambda request:1)
q.feed('a=tool("lookup",key="book")\nb=(\n')
assert q.seen == 0
print(json.dumps({"chunks":observed,"started":started,"claims":claims,"combined_incomplete_seen":q.seen},ensure_ascii=False,indent=2))