# RL 101 hands-on lab: teach a room to cool itself. numpy only, runs offline.
# 2026 CCS Urban AI + Agent Workshop, Harbin Institute of Technology
import numpy as np
np.set_printoptions(precision=2, suppress=True)

# ===== Lab 1: a bandit. Which setpoint gets the fewest complaints? =====
SETPOINTS = [22, 23, 24, 25, 26]
TRUE_OK = np.array([0.55, 0.70, 0.80, 0.72, 0.50])   # hidden: chance an occupant says "comfortable"

def bandit(eps, steps=500, seed=0):
    r = np.random.default_rng(seed)
    n = np.zeros(5); q = np.zeros(5); total = 0
    for t in range(steps):
        a = r.integers(5) if r.random() < eps else int(np.argmax(q))
        reward = float(r.random() < TRUE_OK[a])            # 1 = comfortable vote
        n[a] += 1; q[a] += (reward - q[a]) / n[a]           # running average
        total += reward
    return q, n, total / steps

# ===== Lab 2: the room simulator (a tiny world model) =====
# One office room in a summer day, step = 1 hour, 24 steps per episode.
HOURS = np.arange(24)
T_OUT = 27 + 6 * np.sin((HOURS - 9) / 24 * 2 * np.pi)      # outdoor temperature, peak at 15:00
SUN = np.clip(np.sin((HOURS - 6) / 12 * np.pi), 0, None)   # solar gain 06:00-18:00
OCC = ((HOURS >= 9) & (HOURS < 18)).astype(float)           # people in the office 09:00-18:00
PRICE = np.where((HOURS >= 16) & (HOURS < 21), 3.0, 1.0)    # evening peak price, 16:00-21:00
ACTIONS = [0.0, 1.5, 3.0]                                   # cooling: off, low, high (degrees removed per hour)
POWER = [0.0, 1.0, 2.2]                                     # kWh per hour for each action

def step(T_in, h, a):
    """Next indoor temperature, energy cost and discomfort for one hour."""
    T_next = T_in + 0.15 * (T_OUT[h] - T_in) + 1.2 * SUN[h] + 0.6 * OCC[h] - ACTIONS[a]
    cost = POWER[a] * PRICE[h]
    discomfort = OCC[h] * max(0.0, T_next - 26.0) ** 2 + OCC[h] * max(0.0, 21.0 - T_next) ** 2
    return T_next, cost, discomfort

W = 1.0   # how much one unit of discomfort counts against one unit of cost
def reward(cost, discomfort):
    return -(cost + W * discomfort)

def run(policy, T0=26.0):
    T = T0; trace = []; C = D = 0.0
    for h in range(24):
        a = policy(T, h)
        T, c, d = step(T, h, a)
        C += c; D += d; trace.append((h, a, T))
    return C, D, trace

# ===== Lab 3: the rule an engineer writes =====
def rule(T, h):
    return 2 if (T > 26 and OCC[h]) else (1 if T > 25 and OCC[h] else 0)

# ===== Lab 4: Q-learning =====
def tbin(T):
    return int(np.clip(round((T - 18) / 1.0), 0, 16))       # 17 temperature bins, 18..34 C

def q_learning(episodes=6000, alpha=0.1, gamma=0.95, eps=0.2, seed=0):
    r = np.random.default_rng(seed)
    Q = np.zeros((17, 24, 3))
    for ep in range(episodes):
        T = r.uniform(24, 28)
        e = max(0.02, eps * (1 - ep / episodes))
        for h in range(24):
            s = tbin(T)
            a = r.integers(3) if r.random() < e else int(np.argmax(Q[s, h]))
            T2, c, d = step(T, h, a)
            rew = reward(c, d)
            target = rew + (gamma * Q[tbin(T2), h + 1].max() if h < 23 else 0.0)
            Q[s, h, a] += alpha * (target - Q[s, h, a])     # the Q-learning update
            T = T2
    return Q

if __name__ == '__main__':
    print('Lab 1')
    for eps in (0.0, 0.1, 0.3):
        rs = [bandit(eps, seed=s) for s in range(20)]
        q, n, avg = bandit(eps, seed=0)
        print(f'eps={eps}: mean comfortable votes per step = {np.mean([x[2] for x in rs]):.3f} | seed 0 picks setpoint', SETPOINTS[int(np.argmax(q))], '| tries', n.astype(int))

    print('\nLab 2')
    T = 26.0
    for h in (8, 12, 15, 18):
        print(f'hour {h:02d}: outdoor {T_OUT[h]:.1f} C, sun {SUN[h]:.2f}, people {int(OCC[h])}, price {PRICE[h]:.0f}')
    C, D, tr = run(lambda T, h: 0)
    print('no cooling all day: cost', round(C, 1), '| discomfort', round(D, 1), '| hottest indoor', round(max(t for _, _, t in tr), 1))

    print('\nLab 3')
    C, D, tr = run(rule)
    print('rule-based thermostat: cost', round(C, 1), '| discomfort', round(D, 1), '| total', round(C + W * D, 1))

    print('\nLab 4')
    Q = q_learning()
    pol = lambda T, h: int(np.argmax(Q[tbin(T), h]))
    C2, D2, tr2 = run(pol)
    print('Q-learning agent:      cost', round(C2, 1), '| discomfort', round(D2, 1), '| total', round(C2 + W * D2, 1))
    res = []
    for s in range(5):
        Qs = q_learning(seed=s)
        c, d, _ = run(lambda T, h: int(np.argmax(Qs[tbin(T), h])))
        res.append(c + W * d)
    print('Q-learning total over 5 training seeds: mean', round(float(np.mean(res)), 1), '| min', round(min(res), 1), '| max', round(max(res), 1))

    print('\nLab 5: the agent\'s day, hour by hour (action 0 off, 1 low, 2 high)')
    print('hour  price  rule-action rule-T   agent-action agent-T')
    for (h, a1, t1), (_, a2, t2) in zip(tr, tr2):
        if 6 <= h <= 20:
            print(f'{h:02d}    {PRICE[h]:.0f}      {a1}          {t1:5.1f}    {a2}            {t2:5.1f}')
    peak = sum(POWER[a] for h, a, _ in tr2 if 16 <= h < 21); peak_r = sum(POWER[a] for h, a, _ in tr if 16 <= h < 21)
    print(f'energy 16:00-21:00 (during the peak):  rule {peak_r:.1f} kWh, agent {peak:.1f} kWh')
