# FILE: livingston_be2025.py
"""IMOEXF: midpoint -> breakout extremum, 1+1; common target 1.5 times average-to-stop risk."""
from pathlib import Path
from datetime import datetime, timezone, timedelta
from collections import defaultdict, Counter
import json, math, copy, hashlib
import zone_bars

P=Path(__file__).resolve().parent
OUT=P.parent/'outputs'/'livingston-be-2025'; OUT.mkdir(parents=True,exist_ok=True)
MSK=timezone(timedelta(hours=3)); START='2025-01-01'; END='2025-12-31'
BE_BUFFER=None
def ts(s): return datetime.strptime(s,'%Y-%m-%d %H:%M:%S').replace(tzinfo=MSK).timestamp()
raw=[]
for name in ('imoexf-h1-earlier.json','imoexf-h1-warmup.json','imoexf-h1.json'):
    raw+=json.loads((P/name).read_text(encoding='utf-8'))['data']
raw=sorted({r[-2]:r for r in raw}.values(),key=lambda r:r[-2])
days=defaultdict(list)
for r in raw:days[r[-2][:10]].append(r)
dates=sorted(days); idx={d:i for i,d in enumerate(dates)}
d1=[]; hours=[]
for d in dates:
    a=days[d];d1.append([ts(d+' 08:00:00'),a[0][0],max(r[2] for r in a),min(r[3] for r in a),a[-1][1]])
    for r in a:hours.append(dict(t=r[-2],day=d,j=idx[d],o=r[0],h=r[2],l=r[3],c=r[1]))
events=zone_bars.scan(d1,d1,'D1',since=ts(START+' 00:00:00'))
signals={}
for e in events:
    date=datetime.fromtimestamp(e['t'],MSK).strftime('%Y-%m-%d'); side=e['side']; key=(date,side)
    if key not in signals:
        b=d1[idx[date]]; rng=b[2]-b[3]
        # Keep the exact user-specified midpoint; this version is not tick-rounded.
        ent=(b[2]+b[3])/2
        signals[key]=dict(signal=date,j=idx[date],side=side,kinds=[],zlo=e['zlo'],zhi=e['zhi'],zt=e['zt'],
            low=b[3],high=b[2],signal_open=b[1],bar_range=rng,entry_limit=ent,add_limit=b[2] if side>0 else b[3],
            stop=b[3]-10 if side>0 else b[2]+10)
    s=signals[key];s['kinds'].append(e['kind']);s['zlo']=min(s['zlo'],e['zlo']);s['zhi']=max(s['zhi'],e['zhi'])
signals={k:v for k,v in signals.items() if START<=v['signal']<=END}
for s in signals.values():s['kinds']=sorted(set(s['kinds']))
by_day=defaultdict(list)
for s in signals.values():by_day[s['signal']].append(s)
minute_hours=defaultdict(list)
for p in P.glob('imoexf-m1-*.json'):
    for r in json.loads(p.read_text(encoding='utf-8'))['data']:
        day=r[-2][:10]
        if day not in idx: continue
        minute_hours[r[-2][:13]].append(dict(t=r[-2],day=day,j=idx[day],o=r[0],h=r[2],l=r[3],c=r[1]))

hours=[dict(t=r[-2],day=r[-2][:10],j=idx[r[-2][:10]],o=r[0],h=r[2],l=r[3],c=r[1]) for r in json.loads((P/'imoexf-m10-2025.json').read_text())['data']]

# Replace entire execution days with actual M1 candles after aggregate validation.
for date in sorted({h['day'] for h in hours}):
    mins=sorted([m for group in minute_hours.values() for m in group if m['day']==date],key=lambda m:m['t'])
    if not mins:continue
    base=[h for h in hours if h['day']==date]
    assert [base[0]['o'],max(h['h'] for h in base),min(h['l'] for h in base),base[-1]['c']]==[mins[0]['o'],max(h['h'] for h in mins),min(h['l'] for h in mins),mins[-1]['c']]
    hours=[h for h in hours if h['day']!=date]+mins
hours.sort(key=lambda h:h['t'])

def new_state(s,add):
    return dict(**copy.deepcopy(s),status='pending',fills=[],quantity=0,average=None,target=None,
                initial_stop=s['stop'],be_done=False,be_events=[],add_enabled=add,added=False,turnover=0,ambiguous_hours=0)
def fill(st,price,quantity,t,j):
    st['fills'].append(dict(time=t,price=price,quantity=quantity))
    st['quantity']+=quantity;st['turnover']+=quantity
    st['average']=sum(f['price']*f['quantity'] for f in st['fills'])/st['quantity']
    target=st['average']+st['side']*1.5*abs(st['average']-st['initial_stop'])
    st['target']=target
    st['status']='open'
    if len(st['fills'])==1:st['entry_time']=t;st['entry_j']=j;st['add_deadline']=j+4
    else:st['added']=True
def close(st,price,t,reason):
    st.update(status=reason,exit_price=price,exit_time=t)
    st['pnl_points_contracts']=st['side']*sum((price-f['price'])*f['quantity'] for f in st['fills'])
    st['pnl_rub_gross']=st['pnl_points_contracts']*10
    st['turnover']+=st['quantity']
    st['r_first']=st['pnl_points_contracts']/abs(st['entry_limit']-st['initial_stop'])
    st['risk_points_contracts']=abs(st['average']-st['initial_stop'])*st['quantity']
    st['r']=st['pnl_points_contracts']/st['risk_points_contracts']
def expired(st,h):
    st.update(status='expired',exit_time=h['t'],expiry_reason='5 дневных сессий')
def run_hour(st,h,order):
    st=copy.deepcopy(st); side=st['side'];o=h['o']
    if st['status']=='pending' and h['j']>st['j']+5:expired(st,h);return st
    if st['status']=='open' and h['j']>st['add_deadline']:st['add_enabled']=False
    if st['status'] not in ('pending','open'):return st
    # Existing stops have priority over additions on gaps through the stop.
    if (o-st['stop'])*side<=0:
        if st['status']=='open':close(st,o,h['t'],'stop')
        else:st.update(status='cancelled',exit_time=h['t'],expiry_reason='Открытие за стопом до первого входа')
        return st
    if st['status']=='open' and (o-st['target'])*side>=0:close(st,st['target'],h['t'],'target');return st
    # User's level-touch model: a gap is not an execution at a different price.
    # Both orders fill only at their exact specified level, once price touches it.
    if st['status']=='pending' and abs(o-st['entry_limit'])<1e-9:fill(st,st['entry_limit'],1,h['t'],h['j'])
    if st['status']=='open' and st['add_enabled'] and not st['added'] and abs(o-st['add_limit'])<1e-9:
        fill(st,st['add_limit'],1,h['t'],h['j'])
    if st['status']=='open' and BE_BUFFER is not None and not st['be_done'] and side*(o-st['average'])>=abs(st['average']-st['initial_stop']):
        st['be_done']=True;st['stop']=st['average']*(1+side*BE_BUFFER)
        st['be_events'].append(dict(time=h['t'],trigger=o,stop=st['stop']))
    nodes=[o,h['h'],h['l'],h['c']] if order=='HL' else [o,h['l'],h['h'],h['c']]
    for a,b in zip(nodes,nodes[1:]):
        x=a
        while st['status'] in ('pending','open') and abs(b-x)>1e-9:
            levels=[]
            if st['status']=='pending':levels=[('entry',st['entry_limit'])]
            else:
                levels=[('stop',st['stop']),('target',st['target'])]
                if BE_BUFFER is not None and not st['be_done']:levels.append(('be',st['average']+side*abs(st['average']-st['initial_stop'])))
                if st['add_enabled'] and not st['added']:levels.append(('add',st['add_limit']))
            hits=[(abs(v-x),kind,v) for kind,v in levels if min(x,b)-1e-9<=v<=max(x,b)+1e-9]
            if not hits:break
            _,kind,v=min(hits);x=v
            if kind=='entry':fill(st,v,1,h['t'],h['j'])
            elif kind=='be':
                st['be_done']=True
                st['stop']=st['average']*(1+side*BE_BUFFER)
                st['be_events'].append(dict(time=h['t'],trigger=v,stop=st['stop']))
            elif kind=='add':fill(st,v,1,h['t'],h['j'])
            else:close(st,v,h['t'],kind);break
    return st
def value(st,c):
    if st['status'] in ('stop','target'):return st['pnl_points_contracts']
    return st['side']*sum((c-f['price'])*f['quantity'] for f in st['fills'])
def signature(st):
    return (st['status'],tuple((f['price'],f['quantity']) for f in st['fills']),st['target'],st.get('exit_price'),st['stop'],st['be_done'])
def simulate(add=True,model='adverse'):
    st=None;trades=[];orders=[];skipped=[];curve=[];realized=0;ambiguous=[];resolved=0;unresolved_minutes=0
    test_hours=[h for h in hours if START<=h['day']<=END]
    for n,h in enumerate(test_hours):
        if st and st['status'] in ('pending','open'):
            a=run_hour(st,h,'HL');b=run_hour(st,h,'LH')
            if signature(a)!=signature(b):
                mins=[m for m in minute_hours.get(h['t'][:13],[]) if h['t']<=m['t'] and ts(m['t'])<ts(h['t'])+600]
                record=dict(time=h['t'],signal=st['signal'],side=st['side'],HL=signature(a),LH=signature(b),resolved_by_M1=bool(mins))
                if mins:
                    assert [mins[0]['o'],max(m['h'] for m in mins),min(m['l'] for m in mins),mins[-1]['c']]==[h['o'],h['h'],h['l'],h['c']]
                    chosen=copy.deepcopy(st);resolved+=1
                    for m in mins:
                        ma=run_hour(chosen,m,'HL');mb=run_hour(chosen,m,'LH')
                        if signature(ma)!=signature(mb):unresolved_minutes+=1
                        if model=='HL':chosen=ma
                        elif model=='LH':chosen=mb
                        else:chosen=min((ma,mb),key=lambda q:(value(q,m['c']),0 if q['status']=='stop' else 1))
                elif model=='HL':chosen=a
                elif model=='LH':chosen=b
                else:chosen=min((a,b),key=lambda q:(value(q,h['c']),0 if q['status']=='stop' else 1))
                ambiguous.append(record)
                chosen['ambiguous_hours']=st['ambiguous_hours']+1
            else:chosen=a
            st=chosen
            if st['status'] in ('stop','target'):
                trades.append(st);realized+=st['pnl_points_contracts'];st=None
            elif st['status'] in ('expired','cancelled'):orders.append(st);st=None
        # Decisions only after the final H1 bar of this calendar-day session.
        last=n==len(test_hours)-1 or test_hours[n+1]['day']!=h['day']
        if last:
            if st and st['status']=='pending' and ((h['c']<st['zlo']) if st['side']>0 else (h['c']>st['zhi'])):
                st.update(status='cancelled',exit_time=h['t'],expiry_reason='Зона утратила силу по закрытию D1');orders.append(st);st=None
            candidates=by_day[h['day']]
            if len({s['side'] for s in candidates})>1:
                skipped.extend(dict(**s,skip='Одновременные сигналы в обе стороны') for s in candidates)
            elif candidates:
                if st:skipped.extend(dict(**s,skip='Есть активный ордер или позиция') for s in candidates)
                else:st=new_state(candidates[0],add)
        curve.append(dict(time=h['t'],gross_pnl_points_contracts=realized+(value(st,h['c']) if st else 0)))
    pnl=[t['pnl_points_contracts'] for t in trades];wins=sum(p>0 for p in pnl);gp=sum(p for p in pnl if p>0);gl=-sum(p for p in pnl if p<0)
    peak=0;dd=0
    for q in curve:peak=max(peak,q['gross_pnl_points_contracts']);dd=max(dd,peak-q['gross_pnl_points_contracts'])
    summary=dict(closed=len(trades),wins=wins,losses=len(trades)-wins,win_pct=100*wins/len(trades) if trades else 0,
        adds=sum(t['added'] for t in trades),gross_points_contracts=sum(pnl),gross_rub=sum(pnl)*10,
        open_pnl_points_contracts=value(st,test_hours[-1]['c']) if st else 0,drawdown_points_contracts=dd,
        profit_factor=gp/gl if gl else None,sum_r_first=sum(t['r_first'] for t in trades),
        expired=len(orders),skipped=len(skipped),ambiguous_hours=len(ambiguous),
        turnover_contract_sides=sum(t['turnover'] for t in trades),open_state=st['status'] if st else None,
        resolved_hours_by_M1=resolved,unresolved_minutes=unresolved_minutes)
    summary['after_5rub_per_side']=summary['gross_rub']-5*summary['turnover_contract_sides']
    return dict(summary=summary,trades=trades,orders=orders,skipped=skipped,curve=curve,open=st,ambiguities=ambiguous)

if __name__=='__main__':
    tests={name:simulate(add,model) for name,add,model in [('1plus1_adverse',True,'adverse'),('1plus1_HL',True,'HL'),('1plus1_LH',True,'LH'),('no_add_adverse',False,'adverse')]}
    info=dict(instrument='IMOEXF',start=START,end=END,daily_bars=sum(START<=d<=END for d in dates),
        hourly_bars=sum(START<=h['day']<=END for h in hours),source_signals=len(events),unique_signals=len(signals),
        kinds=dict(Counter(k for s in signals.values() for k in s['kinds'])),signals=list(signals.values()),
        rules_sha256={f:hashlib.sha256((P/f).read_bytes()).hexdigest() for f in ('dzones.py','zone_bars.py')},tests=tests)
    (OUT/'results.json').write_text(json.dumps(info,ensure_ascii=False,indent=2),encoding='utf-8')
    (OUT/'daily-candles.json').write_text(json.dumps([dict(date=dates[i],open=b[1],high=b[2],low=b[3],close=b[4]) for i,b in enumerate(d1)],ensure_ascii=False),encoding='utf-8')
    for k,r in tests.items():print(k,json.dumps(r['summary'],ensure_ascii=False))
    print('signals',len(signals),info['kinds'],'daily',info['daily_bars'])


# FILE: run_be2025.py
import json, importlib, sys, html
from pathlib import Path
from fetch_backtest_data import fetch
import livingston_be2025 as b
variants={'Исходный стоп':None,'БУ после 1R':0,'БУ + 0,1% после 1R':.001}
for attempt in range(4):
    tests={}
    missing=set()
    for name,buffer in variants.items():
        b.BE_BUFFER=buffer
        tests[name]=b.simulate(True)
        missing.update(a['time'][:10] for a in tests[name]['ambiguities'] if not a['resolved_by_M1'])
    if not missing:break
    for day in sorted(missing):fetch(1,day,day,'imoexf-m1-'+day.replace('-','')+'.json')
    b=importlib.reload(b)
for name,result in tests.items():
    trades=result['trades']; s=result['summary'];p=[t['pnl_rub_gross'] for t in trades]
    s['breakeven']=sum(abs(v)<1e-8 for v in p);s['losses']=sum(v<0 for v in p)
    s['sum_r']=sum(t['r'] for t in trades);s['expectancy_r']=s['sum_r']/len(trades) if trades else 0
    s['be_activated']=sum(t['be_done'] for t in trades)
    for cost in (5,10):
        net=[t['pnl_rub_gross']-cost*t['turnover'] for t in trades]
        loss=-sum(v for v in net if v<0)
        s['net_'+str(cost)]=sum(net);s['pf_net_'+str(cost)]=sum(v for v in net if v>0)/loss if loss else None
    for t in trades:
        assert t['entry_time'][:10]>t['signal']
        assert abs(t['target']-(t['average']+t['side']*1.5*abs(t['average']-t['initial_stop'])))<1e-8
        assert len(t['fills'])<=2 and all(f['quantity']==1 for f in t['fills'])
        if t['be_done']:assert t['side']*(t['stop']-t['initial_stop'])>0
    print(name,json.dumps(s,ensure_ascii=False),flush=True)
# Report full sample plus cancelled/skipped campaigns in JSON.
out=b.OUT; (out/'results.json').write_text(json.dumps(dict(period=['2025-01-01','2025-12-31'],signals=list(b.signals.values()),tests=tests),ensure_ascii=False,indent=2),encoding='utf-8')
rows=[]
for name,r in tests.items():
 s=r['summary'];rows.append(f"<tr><td>{name}</td><td>{s['closed']}</td><td>{s['win_pct']:.1f}%</td><td>{s['profit_factor']:.3f}</td><td>{s['gross_rub']:.2f}</td><td>{s['net_5']:.2f}</td><td>{s['net_10']:.2f}</td><td>{s['expectancy_r']:.3f}</td><td>{s['drawdown_points_contracts']*10:.2f}</td><td>{s['be_activated']}</td></tr>")
body=''
for name,r in tests.items():
 body+=f'<h2>{name}</h2><table><tr><th>№</th><th>Сигнал D1</th><th>Направление</th><th>Входы</th><th>Средняя</th><th>Исходный стоп</th><th>Тейк</th><th>Перенос стопа</th><th>Выход</th><th>₽ / R</th></tr>'
 for i,t in enumerate(r['trades'],1):
  fills='<br>'.join(f"{f['time']}: {f['price']:g}" for f in t['fills']);be='<br>'.join(f"{e['time']}: {e['stop']:g}" for e in t['be_events']) or '—'
  body+=f"<tr><td>{i}</td><td>{t['signal']}</td><td>{'LONG ↑' if t['side']>0 else 'SHORT ↓'}</td><td>{fills}</td><td>{t['average']:g}</td><td>{t['initial_stop']:g}</td><td>{t['target']:g}</td><td>{be}</td><td>{t['exit_time']}<br>{t['exit_price']:g} ({t['status']})</td><td>{t['pnl_rub_gross']:.2f} / {t['r']:.3f}</td></tr>"
 body+='</table>'
text='''<!doctype html><meta charset="utf-8"><title>IMOEXF 2025 — безубыток</title><style>body{font:16px system-ui;max-width:1400px;margin:30px auto;padding:15px;color:#18352b}table{border-collapse:collapse;font-size:13px}td,th{padding:9px;border:1px solid #ddd;text-align:left}th{background:#e9f3ee}</style><h1>IMOEXF: перенос стопа после +1R, 2025</h1><p>Полный календарный 2025 год. D1 сигналы, исполнение M10, неоднозначные участки M1. Первый вход — середина D1, второй — пробой экстремума, по 1 контракту. Исходный стоп — экстремум ±10 пунктов. Тейк 1,5R от средней цены; R измеряется до исходного стопа. Тейк после переноса стопа не приближается. БУ +0,1% считается от средней цены в сторону прибыли. Все кампании независимы между вариантами; внутри варианта одна активная кампания.</p><p>Цены теоретические, без округления к тику. Комиссия условная: 5 ₽ за контракт за сторону; второй сценарий 10 ₽ включает дополнительно 5 ₽ на сторону на проскальзывание. Фандинг и дивидендные корректировки не включены. Просадка по закрытиям M10, до издержек. Это сравнение на истории, не доказательство будущей доходности.</p><table><tr><th>Вариант</th><th>Сделок</th><th>Побед</th><th>PF до затрат</th><th>₽ до затрат</th><th>₽ комиссия</th><th>₽ комиссия + проскальзывание</th><th>Среднее R</th><th>Просадка ₽</th><th>Переносов</th></tr>'''+''.join(rows)+'</table>'+body+'<p><a href="results.json">Полные результаты, сигналы и статусы JSON</a></p>'
(out/'report.html').write_text(text,encoding='utf-8')



# FILE: zone_bars.py
"""Closed-candle patterns at D1 zones; historical scans never use future zones."""
import bisect
import dzones as Z
LABELS = {'outside': 'Внешний бар', 'exhaustion': 'Бар истощения'}

def scan(d1, bars, tf, exhaustion=True, since=None, current_only=False):
    if len(d1)<5 or len(bars)<2: return []
    dts=[b[0] for b in d1]
    raw=Z.zones(d1)
    if current_only:
        active={(z["side"],z["t0"]) for z in Z.active_zones(d1)}
        raw=[z for z in raw if (z[0],z[3]) in active]
    out=[]
    for i in range(1,len(bars)):
        t,o,h,l,c=bars[i][:5]
        if since is not None and t<since: continue
        if h<=l: continue
        # D1 timestamps are opening times. Only prior daily bars are known
        # at the open of a signal bar, also on H1/H4 and across session gaps.
        j=bisect.bisect_right(dts,t)-1
        known=j-1
        kinds=[]
        body=abs(c-o); upper=h-max(o,c); lower=min(o,c)-l
        if h>bars[i-1][2] and l<bars[i-1][3] and body>=0.25*(h-l): kinds.append('outside')
        is_exhaustion=exhaustion and body>0 and body>=0.15*(h-l)
        if not kinds and not is_exhaustion: continue
        for side,lo,hi,t0,k in raw:
            if not(k<=known<=k+250): continue
            if any(x[4]>hi if side<0 else x[4]<lo for x in d1[k+1:known+1]): continue
            # An invalidating D1 close is not a reversal signal at a live zone.
            if tf=='D1' and (c>hi if side<0 else c<lo): continue
            if h<lo or l>hi: continue
            event_kinds=kinds[:]
            if is_exhaustion and ((side<0 and upper>=1.5*body and lower<=0.5*body) or (side>0 and lower>=1.5*body and upper<=0.5*body)):
                event_kinds.append('exhaustion')
            for kind in event_kinds:
                out.append(dict(t=t,tf=tf,kind=kind,label=LABELS[kind],side=side,zlo=lo,zhi=hi,zt=t0,close=c))
    # Overlapping source zones produce one marker/notification per kind and side.
    unique={}
    for e in out:
        key=(e['t'],e['kind'],e['side'])
        if key in unique:
            unique[key]['zlo']=min(unique[key]['zlo'],e['zlo']);unique[key]['zhi']=max(unique[key]['zhi'],e['zhi'])
        else: unique[key]=e
    return list(unique.values())


def latest(data):
    import time
    out=[]
    for tf in ('H1','H4','D1'):
        bars=data.get(tf,[])
        if not bars or time.time()-bars[-1][0]>5*86400: continue
        out.extend(scan(data.get('D1',[]),bars,tf,since=bars[-1][0],current_only=True))
    return out


def chart(bars, event, name, path):
    import matplotlib
    matplotlib.use('Agg')
    import matplotlib.pyplot as plt
    from matplotlib.patches import Rectangle
    from datetime import datetime, timezone, timedelta
    seg=[b for b in bars if b[0]<=event['t']][-90:]
    fig,ax=plt.subplots(figsize=(12,6),dpi=120)
    fig.patch.set_facecolor('#161b24');ax.set_facecolor('#161b24')
    color='#22c55e' if event['side']>0 else '#ef4444'
    ax.axhspan(event['zlo'],event['zhi'],color=color,alpha=.2)
    span=max(b[2] for b in seg)-min(b[3] for b in seg)
    for i,b in enumerate(seg):
        t,o,h,l,c=b[:5];col='#22c55e' if c>=o else '#ef4444'
        ax.vlines(i,l,h,color=col,linewidth=1)
        ax.add_patch(Rectangle((i-.32,min(o,c)),.64,max(abs(c-o),span*.001),color=col))
    ax.scatter([len(seg)-1],[seg[-1][3] if event['side']>0 else seg[-1][2]],marker='^' if event['side']>0 else 'v',s=110,color='#fbbf24',zorder=5)
    ax.set_title(f"{name} · {event['tf']} · {event['label']}",color='white')
    ax.tick_params(colors='#a8b3c7');ax.yaxis.tick_right();ax.grid(alpha=.15)
    for sp in ax.spines.values():sp.set_color('#3b4555')
    ticks=list(range(0,len(seg),max(1,len(seg)//6)))
    ax.set_xticks(ticks,[datetime.fromtimestamp(seg[i][0],timezone.utc).astimezone(timezone(timedelta(hours=3))).strftime('%d.%m %H:%M') for i in ticks],fontsize=8)
    fig.tight_layout();fig.savefig(path);plt.close(fig)


def tick(baseline=False):
    import json,os,time
    from pathlib import Path
    from datetime import datetime,timezone,timedelta
    import db,worker,structure
    candles={}
    for filename in ('candles.json','candles_ext.json'):
        try: candles.update(json.loads(Path(db.DATA_DIR,filename).read_text()))
        except (OSError,ValueError): pass
    statepath=Path(db.DATA_DIR,'dz_bar_state.json')
    try: state=json.loads(statepath.read_text())
    except (OSError,ValueError): state={}
    subs=worker.subscribers('dz_sub')
    promo=time.time()<Z.FREE_UNTIL
    insts={i['id'].rstrip('+'):i for i in db.INSTRUMENTS+structure.EXTRA_INSTRUMENTS}
    for sym,inst in insts.items():
        data=candles.get(sym,{})
        for tf in ('H1','H4','D1'):
            bars=data.get(tf,[]);key=f'{sym}|{tf}'
            if not bars:continue
            last=bars[-1][0];old=state.get(key)
            if old==last:continue
            if old is None or baseline or time.time()-last>5*86400:
                state[key]=last;continue
            events=scan(data.get('D1',[]),bars,tf,since=old+1,current_only=True)
            to=[]
            for uid,scope in subs:
                if scope=='imoexf' and not promo and inst['id']!='IMOEXF':continue
                chosen=db.kv_get(f'dz_syms:{uid}')
                if chosen is not None:
                    if inst['id'] not in json.loads(chosen):continue
                else:
                    with db.conn() as con:
                        admin=con.execute('SELECT is_admin FROM users WHERE id=?',(uid,)).fetchone()[0]
                    if not admin and inst['id'] not in ('GAZP','LKOH','OZON','SBER','ROSN','SMLT','NVTK','T','PLZL','GMKN'):continue
                to.append(uid)
            for event in events:
                if not to:continue
                dt=datetime.fromtimestamp(event['t'],timezone.utc).astimezone(timezone(timedelta(hours=3))).strftime('%d.%m.%Y %H:%M')
                side='спроса' if event['side']>0 else 'предложения'
                cap=f"🔔 {inst['nm']} · {event['tf']}\n{event['label']} в зоне {side} D1\nСвеча: {dt} МСК (время открытия, сигнал после закрытия)\nЗона: {Z.fmt(event['zlo'])} – {Z.fmt(event['zhi'])}\nЦена закрытия: {Z.fmt(event['close'])}"
                if event['kind']=='exhaustion':cap+='\nРабочая тень ≥ 1,5 тела; противоположная ≤ 0,5 тела; тело ≥ 15% диапазона.'
                path=str(Path(db.DATA_DIR,'dz_bar_alert.png'))
                chart(bars,event,inst['nm'],path)
                kb=json.dumps({'inline_keyboard':[[{'text':'Открыть график','url':f'https://panovq.trade/dzones?s={inst["id"]}&tf={tf}&from=tg'}]]})
                worker.send_photo_many(to,path,cap,kb)
            state[key]=last
    tmp=statepath.with_suffix('.tmp');tmp.write_text(json.dumps(state));os.replace(tmp,statepath)


# FILE: dzones.py
"""Зоны D1 + пин-бар H4 (система пользователя, проверка 2026-10-09: лучший вариант — H4, вход от максимума тени, тейк 1:5).

Зона предложения (D1): фрактал 2+2 по максимуму; зона = [max(open, close); high] — тень дневного разворота (ложные пробои).
  Известна после закрытия 2-й свечи справа; действует, пока дневная свеча не закроется за ней, не дольше 250 дней. Спрос — зеркально.
Сигнал (H4): свеча заходит в зону, диапазон ≥ ATR(14) H4, рабочая тень ≥ 3 тел, обратная тень ≤ тела и ≤ 25% диапазона.
Ордер: лимит от экстремума тени, действует 5 свечей H4; стоп = длина тени от входа; тейк = 5 стопов.
Статусы: order (ордер выставлен) → open (исполнен) → win / loss; expired — не исполнился за 5 свечей.
Свечи — из data/candles.json (только закрытые). Без зависимостей от web.py."""
import bisect, os
from datetime import datetime, timezone, timedelta

K_TP = 5
WICK_K = 3.0
EXPIRE = 5
STATUS = {"order": "Ордер выставлен", "open": "В сделке", "win": "Тейк 1:5", "loss": "Стоп", "expired": "Не исполнился"}


def _atr(b, n=14):
    out = [None] * len(b)
    trs = []
    for i in range(1, len(b)):
        h, l, pc = b[i][2], b[i][3], b[i - 1][4]
        trs.append(max(h - l, abs(h - pc), abs(l - pc)))
        if len(trs) >= n:
            out[i] = sum(trs[-n:]) / n
    return out


def zones(d1):
    """[(side, lo, hi, t_bar, k_known)] side=-1 предложение, +1 спрос; k_known — индекс дня, с закрытия которого зона известна."""
    Z = []
    for i in range(2, len(d1) - 2):
        o, h, l, c = d1[i][1:5]
        if h > max(x[2] for x in d1[i - 2:i]) and h > max(x[2] for x in d1[i + 1:i + 3]):
            Z.append((-1, max(o, c), h, d1[i][0], i + 2))
        if l < min(x[3] for x in d1[i - 2:i]) and l < min(x[3] for x in d1[i + 1:i + 3]):
            Z.append((1, l, min(o, c), d1[i][0], i + 2))
    return Z


def active_zones(d1):
    """Зоны, действующие на текущий момент (после последней закрытой дневной свечи)."""
    out = []
    n = len(d1)
    for side, lo, hi, t0, k in zones(d1):
        if n - 1 - k > 250:
            continue
        broken = any((x[4] > hi) if side < 0 else (x[4] < lo) for x in d1[k + 1:])
        if not broken:
            out.append(dict(side=side, lo=lo, hi=hi, t0=t0))
    return out


def _day(t):
    return int(t // 86400)


def scan(d1, h4, since=None):
    """Сигналы H4 у действующих зон D1. since — не раньше этого времени (сек). Возвращает список сигналов со статусом."""
    if len(d1) < 30 or len(h4) < 30:
        return []
    Z = zones(d1)
    dts = [b[0] for b in d1]
    A = _atr(h4)
    sigs, busy = [], -1
    for i in range(15, len(h4)):
        t, o, h, l, c = h4[i][:5]
        if since and t < since:
            continue
        if i <= busy or A[i - 1] is None:
            continue
        rng = h - l
        if rng <= 0 or rng < A[i - 1]:
            continue
        j = bisect.bisect_right(dts, t) - 1          # дневная свеча, внутри которой эта H4
        if j < 0:
            continue
        body = abs(c - o); bodyc = max(body, rng * 0.02)
        up, dn = h - max(o, c), min(o, c) - l
        for side, lo, hi, t0, k in Z:
            if not (k < j <= k + 250):  # зона известна к закрытию дня k, текущий день j позже
                continue
            if any((x[4] > hi) if side < 0 else (x[4] < lo) for x in d1[k + 1:j]):
                continue
            if not (h >= lo and l <= hi):
                continue
            if side < 0 and up >= WICK_K * bodyc and dn <= bodyc and dn <= 0.25 * rng:
                wick, ent = up, h
            elif side > 0 and dn >= WICK_K * bodyc and up <= bodyc and up <= 0.25 * rng:
                wick, ent = dn, l
            else:
                continue
            stop = ent - side * wick
            tp = ent + side * K_TP * wick
            s = dict(t=t, i=i, side=side, body=bodyc, entry=ent, stop=stop, target=tp, wick=wick, zlo=lo, zhi=hi, zt=t0, status="order")
            _track(s, h4)
            sigs.append(s)
            busy = s.get("i_in", i + EXPIRE) if s["status"] != "order" else len(h4)
            break
    return sigs


def _track(s, h4):
    i, side, E, S, T = s["i"], s["side"], s["entry"], s["stop"], s["target"]
    jin = None
    for q in range(i + 1, min(len(h4), i + 1 + EXPIRE)):
        if (h4[q][2] >= E) if side < 0 else (h4[q][3] <= E):
            jin = q; break
    if jin is None:
        if len(h4) - 1 >= i + EXPIRE:
            s["status"] = "expired"; s["t_out"] = h4[i + EXPIRE][0]
        return
    s.update(status="open", i_in=jin, t_in=h4[jin][0])
    for q in range(jin, len(h4)):
        hi, lo = h4[q][2], h4[q][3]
        if (lo <= S) if side > 0 else (hi >= S):
            s.update(status="loss", t_out=h4[q][0], r=-1.0); return
        if q > jin and ((hi >= T) if side > 0 else (lo <= T)):
            s.update(status="win", t_out=h4[q][0], r=float(K_TP)); return
    last = h4[-1][4]
    s["r_now"] = round((last - E) * side / abs(E - S), 2)


def fmt(v):
    return (f"{v:,.4f}" if abs(v) < 5 else f"{v:,.2f}" if abs(v) < 1000 else f"{v:,.1f}").replace(",", " ").replace(".", ",")


def chart(sym, nm, s, h4, path):
    """PNG: H4 вокруг сигнала, зона D1, вход/стоп/тейк, исход. Подписи — плашками справа."""
    import matplotlib
    matplotlib.use("Agg")
    import matplotlib.pyplot as plt
    from matplotlib.patches import Rectangle
    MSK = timedelta(hours=3)
    INK, MUT, GOOD, BAD, ACC, ZON = "#14203a", "#5d6883", "#0b7a55", "#b42318", "#2446c7", "#f5b400"
    i = s["i"]; b0 = max(0, i - 50); b1 = len(h4)
    if s.get("t_out"):
        jx = next((q for q in range(i, len(h4)) if h4[q][0] >= s["t_out"]), len(h4) - 1); b1 = min(len(h4), jx + 8)
    b0 = max(0, min(b0, b1 - 70)); seg = h4[b0:b1]; n = len(seg)
    side, E, S, T = s["side"], s["entry"], s["stop"], s["target"]
    fig = plt.figure(figsize=(12, 6.6), dpi=100); fig.patch.set_facecolor("white")
    ax = fig.add_axes([0.04, 0.1, 0.72, 0.72])
    for sp in ax.spines.values(): sp.set_visible(False)
    ax.tick_params(colors=INK, labelsize=10.5, length=0)
    ax.axhspan(s["zlo"], s["zhi"], color=ZON, alpha=0.18, zorder=0)
    for k, (t, o, h, l, c) in enumerate(x[:5] for x in seg):
        col = "#16a34a" if c >= o else "#dc2626"
        if b0 + k == i:
            ax.add_patch(Rectangle((k - 0.6, l), 1.2, h - l, color="#ddd6fe", zorder=0))
        ax.plot([k, k], [l, h], color=col, lw=1)
        ax.add_patch(Rectangle((k - 0.33, min(o, c)), 0.66, max(abs(c - o), abs(E - S) * 0.02), color=col))
    xi = i - b0
    for v, col in ((E, ACC), (S, BAD), (T, GOOD)):
        ax.plot([xi, n], [v, v], color=col, lw=1.6 if v != T else 1.2, ls="-" if v != T else "--")
    if s.get("t_in"):
        xin = next(k for k in range(n) if seg[k][0] >= s["t_in"])
        ax.scatter([xin], [E], marker="v" if side < 0 else "^", s=140, color=ACC, zorder=5)
    if s["status"] in ("win", "loss"):
        xo = next((k for k in range(n) if seg[k][0] >= s["t_out"]), n - 1)
        ax.scatter([xo], [T if s["status"] == "win" else S], s=130, facecolor="white", edgecolor=GOOD if s["status"] == "win" else BAD, lw=2.5, zorder=6)
    lo = min(min(x[3] for x in seg), S, T, s["zlo"]); hi = max(max(x[2] for x in seg), S, T, s["zhi"]); pad = (hi - lo) * 0.05
    ax.set_ylim(lo - pad, hi + pad); ax.set_xlim(-1, n + 1)
    ax.yaxis.tick_right(); ax.grid(True, axis="y", color="#edf0f6")
    ticks = list(range(0, n, max(1, n // 7)))
    ax.set_xticks(ticks); ax.set_xticklabels([(datetime.fromtimestamp(seg[k][0], timezone.utc) + MSK).strftime("%d.%m %H:%M") for k in ticks], fontsize=9.5, color=MUT)
    tr, inv = ax.transData, fig.transFigure.inverted()
    labs = sorted([(E, f"Вход {fmt(E)}", ACC), (S, f"Стоп {fmt(S)}", BAD), (T, f"Тейк 1:5 {fmt(T)}", GOOD),
                   ((s["zlo"] + s["zhi"]) / 2, f"Зона D1 от {(datetime.fromtimestamp(s['zt'], timezone.utc)):%d.%m.%y}", "#a16207")], key=lambda z: z[0])
    ys = [inv.transform(tr.transform((n, v)))[1] for v, _, _ in labs]; adj = list(ys)
    for q in range(1, len(adj)):
        if adj[q] - adj[q - 1] < 0.055: adj[q] = adj[q - 1] + 0.055
    for (v, t, col), y0, y1 in zip(labs, ys, adj):
        fig.text(0.83, y1, t, fontsize=10.5, color="white", fontweight="bold", va="center", bbox=dict(boxstyle="round,pad=0.35", fc=col, ec="none"))
        fig.add_artist(matplotlib.lines.Line2D([0.77, 0.825], [y0, y1], color=col, lw=0.8, transform=fig.transFigure))
    st = STATUS[s["status"]] + (f" ({s['r']:+.0f}R)" if "r" in s else "")
    fig.text(0.04, 0.945, f"{nm}: {'продажа от зоны предложения' if side < 0 else 'покупка от зоны спроса'} — {st}", fontsize=14.5, fontweight="bold",
             color=GOOD if s["status"] == "win" else BAD if s["status"] == "loss" else INK)
    fig.text(0.04, 0.905, f"Таймфрейм: H4 (зона — с D1). Сигнальная свеча {(datetime.fromtimestamp(s['t'], timezone.utc) + MSK):%d.%m %H:%M} МСК — фиолетовый фон; "
             f"тень {fmt(s['wick'])} = {s['wick'] / s['body']:.1f} тела", fontsize=10, color=MUT)
    fig.text(0.04, 0.03, "Лимит от экстремума тени (5 свечей), стоп = длина тени, тейк 5 стопов. Жёлтая полоса — зона D1. panovq.trade", fontsize=9.5, color=MUT)
    os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
    fig.savefig(path, facecolor="white"); plt.close(fig)
    return path


# ---------- Зоны на D1 для отрисовки и уведомлений о входе цены в зону (2026-10-09) ----------
def merged_zones(d1, max_age=250):
    """Действующие зоны, слитые по стороне: пересекающиеся области теней → одна зона, касаний = число разворотов в ней."""
    raw = [z for z in active_zones(d1) if (d1[-1][0] - z["t0"]) / 86400 <= max_age * 1.45]
    out = []
    for side in (-1, 1):
        zs = sorted([z for z in raw if z["side"] == side], key=lambda z: z["lo"])
        cur = None
        for z in zs:
            if cur and z["lo"] <= cur["hi"]:
                cur["hi"] = max(cur["hi"], z["hi"]); cur["n"] += 1; cur["t0"] = min(cur["t0"], z["t0"]); cur["t_last"] = max(cur["t_last"], z["t0"])
            else:
                cur = dict(side=side, lo=z["lo"], hi=z["hi"], n=1, t0=z["t0"], t_last=z["t0"]); out.append(cur)
    # минимальная высота зоны — 0,15 ATR (иначе это линия, а не зона)
    a = (_atr(d1)[-1] or 0) * 0.15
    for z in out:
        if z["hi"] - z["lo"] < a:
            m = (z["hi"] + z["lo"]) / 2; z["lo"], z["hi"] = m - a / 2, m + a / 2
    return out


def zone_key(sym, z):
    return f"{sym}|{z['side']}|{round(z['lo'], 6)}"


def d1_chart(sym, nm, d1, zs, path, last=None, hit=None, n_bars=120):
    """Дневной график с зонами: предложение — красные, спрос — зелёные; в зоне сейчас — ярче. Подписи плашками справа."""
    import matplotlib
    matplotlib.use("Agg")
    import matplotlib.pyplot as plt
    from matplotlib.patches import Rectangle
    MSK = timedelta(hours=3)
    INK, MUT, UP, DN = "#14203a", "#5d6883", "#0b7a55", "#b42318"
    seg = d1[-n_bars:]; n = len(seg)
    last = last if last is not None else seg[-1][4]
    lo0, hi0 = min(x[3] for x in seg), max(x[2] for x in seg)
    span = hi0 - lo0
    vis = [z for z in zs if z["hi"] >= lo0 - span * 0.08 and z["lo"] <= hi0 + span * 0.08]
    above = sorted([z for z in vis if z["lo"] > last], key=lambda z: z["lo"])[:4]
    below = sorted([z for z in vis if z["hi"] < last], key=lambda z: -z["hi"])[:4]
    vis = above + below + [z for z in vis if z["lo"] <= last <= z["hi"]]
    fig = plt.figure(figsize=(12, 6.8), dpi=100); fig.patch.set_facecolor("white")
    ax = fig.add_axes([0.04, 0.09, 0.66, 0.75])
    for sp in ax.spines.values(): sp.set_visible(False)
    ax.tick_params(colors=INK, labelsize=10.5, length=0)
    for z in vis:
        x0 = next((k for k in range(n) if seg[k][0] >= z["t0"]), 0)
        inside = z["lo"] <= last <= z["hi"] or (hit and zone_key(sym, z) == hit)
        col = DN if z["side"] < 0 else UP
        ax.add_patch(Rectangle((x0 - 0.5, z["lo"]), n + 1 - x0, z["hi"] - z["lo"], color=col, alpha=0.32 if inside else 0.13, lw=0, zorder=0))
    for k, (t, o, h, l, c) in enumerate(x[:5] for x in seg):
        col = "#16a34a" if c >= o else "#dc2626"
        ax.plot([k, k], [l, h], color=col, lw=1)
        ax.add_patch(Rectangle((k - 0.33, min(o, c)), 0.66, max(abs(c - o), span * 0.001), color=col))
    ax.axhline(last, color="#2446c7", lw=0.9, ls="--")
    pad = span * 0.08
    ax.set_ylim(lo0 - pad, hi0 + pad); ax.set_xlim(-1, n + 1)
    ax.yaxis.tick_right(); ax.grid(True, axis="y", color="#edf0f6")
    ticks = list(range(0, n, max(1, n // 7)))
    ax.set_xticks(ticks); ax.set_xticklabels([(datetime.fromtimestamp(seg[k][0], timezone.utc) + MSK).strftime("%d.%m.%y") for k in ticks], fontsize=9.5, color=MUT)
    tr, inv = ax.transData, fig.transFigure.inverted()
    labs = []
    for z in vis:
        m = min(max((z["lo"] + z["hi"]) / 2, lo0 - pad), hi0 + pad)
        inside = z["lo"] <= last <= z["hi"]
        t = ("◉ " if inside else "") + f"{'Предложение' if z['side'] < 0 else 'Спрос'} {fmt(z['lo'])}–{fmt(z['hi'])}" + (f" ×{z['n']}" if z["n"] > 1 else "")
        labs.append((m, t, DN if z["side"] < 0 else UP, inside))
    labs.append((last, f"Цена {fmt(last)}", "#2446c7", True))
    labs.sort(key=lambda x: x[0])
    ys = [inv.transform(tr.transform((n, v)))[1] for v, *_ in labs]; adj = list(ys)
    for q in range(1, len(adj)):
        if adj[q] - adj[q - 1] < 0.045: adj[q] = adj[q - 1] + 0.045
    over = max(0, adj[-1] - 0.86) if adj else 0
    adj = [y - over for y in adj]
    for (v, t, col, strong), y0, y1 in zip(labs, ys, adj):
        fig.text(0.765, y1, t, fontsize=9.6 if not strong else 10.2, color="white", fontweight="bold", va="center",
                 bbox=dict(boxstyle="round,pad=0.3", fc=col, ec="none", alpha=1 if strong else 0.75))
        fig.add_artist(matplotlib.lines.Line2D([0.705, 0.76], [y0, y1], color=col, lw=0.7, transform=fig.transFigure))
    fig.text(0.04, 0.945, f"{nm}: зоны спроса и предложения", fontsize=15, fontweight="bold", color=INK)
    fig.text(0.04, 0.905, f"D1 · дневной. Красное — предложение, зелёное — спрос. ◉ — цена в зоне, ×N — разворотов. Свеча "
             f"{(datetime.fromtimestamp(seg[-1][0], timezone.utc) + MSK):%d.%m.%Y}", fontsize=10, color=MUT)
    fig.text(0.04, 0.025, "Зона действует, пока дневная свеча не закроется за ней. panovq.trade/dzones", fontsize=9.5, color=MUT)
    os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
    fig.savefig(path, facecolor="white"); plt.close(fig)
    return path


# Акция: зоны D1 бесплатно всем 30 дней (2026-10-09 … 2026-11-08 включительно, МСК)
FREE_UNTIL = 1794171600  # 2026-11-09 00:00 МСК


# FILE: fetch_backtest_data.py
import json, urllib.request, urllib.parse, time
from pathlib import Path
ROOT=Path(__file__).resolve().parent
BASE='https://iss.moex.com/iss/engines/futures/markets/forts/securities/IMOEXF/candles.json'
def fetch(interval,start,end,name):
    target=ROOT/name
    if target.exists():
        return json.loads(target.read_text(encoding='utf-8'))
    rows=[]; offset=0
    while True:
        qs=urllib.parse.urlencode({'from':start,'till':end,'interval':interval,'start':offset,'iss.meta':'off'})
        for attempt in range(4):
            try:
                with urllib.request.urlopen(BASE+'?'+qs,timeout=40) as r: block=json.load(r)['candles']
                break
            except Exception:
                if attempt==3: raise
                time.sleep(1+attempt)
        batch=block['data']
        if not batch: break
        rows.extend(batch);offset+=len(batch)
        print(name,offset,batch[-1][-2],flush=True)
    unique={r[-2]:r for r in rows}
    obj={'columns':block['columns'],'data':[unique[k] for k in sorted(unique)],'source':BASE,'interval':interval,'from':start,'till':end}
    target.write_text(json.dumps(obj,ensure_ascii=False),encoding='utf-8')
    return obj
if __name__=='__main__':
    fetch(24,'2023-11-14','2026-10-12','imoexf-d1.json')
    fetch(60,'2025-10-13','2026-10-12','imoexf-h1.json')