"""
Reading the City from Reviews: hands-on lab with Singapore Google Maps reviews.
Needs: pandas, numpy, scikit-learn, matplotlib (optional, for maps).
Data: sg_places.csv and sg_reviews.csv (a 40,000-review class sample; for teaching only, do not republish).
Run:  python reviews_lab.py
"""
import re
import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split

places = pd.read_csv('sg_places.csv')
reviews = pd.read_csv('sg_reviews.csv', parse_dates=['date'])
df = reviews.merge(places[['place_id', 'name', 'category', 'lat', 'lon']], on='place_id')
low = df.text.str.lower()

# ---------- Lab 2: words that go with good and bad atmosphere ----------
def log_odds(texts_a, texts_b, min_count=30):
    """Which words are much more common in group a than in group b (smoothed log ratio)."""
    cv = CountVectorizer(stop_words='english', min_df=min_count, token_pattern=r'(?u)\b[a-z][a-z]+\b')
    X = cv.fit_transform(pd.concat([texts_a, texts_b]))
    a = np.asarray((X[:len(texts_a)] > 0).sum(0)).ravel() + 1
    b = np.asarray((X[len(texts_a):] > 0).sum(0)).ravel() + 1
    lo = np.log(a / len(texts_a)) - np.log(b / len(texts_b))
    words = np.array(cv.get_feature_names_out())
    order = np.argsort(lo)
    return list(words[order[::-1][:12]]), list(words[order[:12]])

# ---------- Lab 3: from text to a score ----------
def text_model(target='atmosphere', seed=0):
    d = df[df[target].isin([1, 2, 4, 5])].copy()
    d['good'] = (d[target] >= 4).astype(int)
    tr, te = train_test_split(d, test_size=0.25, random_state=seed, stratify=d.good)
    vec = TfidfVectorizer(ngram_range=(1, 2), min_df=3, sublinear_tf=True)
    Xtr, Xte = vec.fit_transform(tr.text), vec.transform(te.text)
    clf = LogisticRegression(max_iter=2000, class_weight='balanced').fit(Xtr, tr.good)
    pred = clf.predict(Xte)
    acc = (pred == te.good).mean()
    bal = 0.5 * ((pred[te.good == 1] == 1).mean() + (pred[te.good == 0] == 0).mean())
    base = max(te.good.mean(), 1 - te.good.mean())
    return vec, clf, len(tr), len(te), base, acc, bal

# ---------- Lab 4: a comfort lens ----------
COMFORT = {
    'heat':  r"\b(?:stuffy|humid|sweat\w*|sweltering|warm inside|hot inside|too warm|very warm|so warm|no air ?-?con\w*|non[- ]air ?-?con\w*|without air ?-?con\w*|poor ventilation|not well ventilated|no ventilation|aircon (?:is|was) (?:weak|not working|not cold))",
    'cool':  r"\b(?:air ?-?con\w*|air conditioned|air conditioning|cooling|well ventilated|breezy|nice and cool|cool and)",
    'noise': r"\b(?:noisy|loud|noise|hard to hear|deafening|blasting)",
    'quiet': r"\b(?:quiet|peaceful|calm|serene|tranquil)",
    'crowd': r"\b(?:crowded|packed|long queue|long line|squeeze|cramped|no seats|hard to find (?:a )?seat)",
    'space': r"\b(?:spacious|roomy|lots of space|plenty of seat\w*|ample seat\w*)",
}
def tag(texts, pattern):
    return texts.str.contains(pattern, regex=True)

FOOD_WORDS = r"\b(?:food|soup|tea|coffee|chocolate|drink|dish|pot|plate|sauce|chilli|chili|spicy|fresh|served|meal|noodle|rice|latte|dog|wings|cake)\b"

if __name__ == '__main__':
    print('\n=== Lab 1: look at the data ===')
    print(f'places: {len(places)} | reviews: {len(reviews)} | from {reviews.date.min():%Y-%m} to {reviews.date.max():%Y-%m}')
    print('top categories:', df.category.value_counts().head(6).to_dict())
    print('median review length:', int(df.text.str.len().median()), 'characters')
    for c in ['food', 'service', 'atmosphere']:
        v = df[c].dropna()
        print(f'{c:10s} scores given in {len(v)/len(df):.0%} of reviews | mean {v.mean():.2f} | share of 1-2 stars {(v <= 2).mean():.0%}')
    print('example:', df.text.iloc[3][:160], '...')

    print('\n=== Lab 2: words of good and bad atmosphere ===')
    good, bad = log_odds(low[df.atmosphere == 5], low[df.atmosphere <= 2])
    print('more common when atmosphere = 5  :', ', '.join(good))
    print('more common when atmosphere <= 2 :', ', '.join(bad))

    print('\n=== Lab 3: predict a good or bad atmosphere from the text ===')
    vec, clf, ntr, nte, base, acc, bal = text_model('atmosphere')
    print(f'train {ntr} | test {nte} | always-guess-good baseline {base:.2f} | model accuracy {acc:.2f} | balanced accuracy {bal:.2f}')
    w = clf.coef_[0]; names = np.array(vec.get_feature_names_out()); o = np.argsort(w)
    print('words pushing toward GOOD:', ', '.join(names[o[::-1][:10]]))
    print('words pushing toward BAD :', ', '.join(names[o[:10]]))
    tests = ['Cosy little cafe, quiet music, great for working.',
             'Very stuffy and noisy, no aircon, tables were dirty.',
             'Food was ok but the place is cramped and hot.']
    for s, pr in zip(tests, clf.predict_proba(vec.transform(tests))[:, 1]):
        print(f'  P(good atmosphere) = {pr:.2f}  <- "{s}"')
    for t in ['service', 'food']:
        _, _, _, _, b, a, ba = text_model(t)
        print(f'same model for {t:8s}: baseline {b:.2f} | accuracy {a:.2f} | balanced {ba:.2f}')

    print('\n=== Lab 4: a comfort lens (word lists) ===')
    hot = tag(low, r'\bhot\b')
    print(f'reviews containing "hot": {hot.sum()} | of those, {tag(low[hot], FOOD_WORDS).mean():.0%} also talk about food or drinks')
    print('   e.g.', [re.search(r'.{0,30}\bhot\b.{0,30}', t).group(0) for t in low[hot].head(3)])
    for k, pat in COMFORT.items():
        df[k] = tag(low, pat)
    print('share of reviews per comfort theme:', {k: f'{df[k].mean():.1%}' for k in COMFORT})
    # check the noise list against the reviewers' own noise labels
    lab = df[df.noise_level.notna()]
    loud = lab.noise_level.isin(['Loud, but you can still talk', 'Very loud, hard to hear'])
    print(f'check against {len(lab)} reviews with a noise label: "noise" words appear in {lab.noise[loud].mean():.0%} of loud vs {lab.noise[~loud].mean():.0%} of quieter places; '
          f'"quiet" words in {lab.quiet[~loud].mean():.0%} of quieter vs {lab.quiet[loud].mean():.0%} of loud')
    a = df.dropna(subset=['atmosphere'])
    for k in ['heat', 'noise', 'crowd', 'quiet', 'space']:
        print(f'  atmosphere when "{k}" is mentioned: {a[a[k]].atmosphere.mean():.2f} (n={a[k].sum()}) vs not: {a[~a[k]].atmosphere.mean():.2f}')
    print('heat complaints by type of place:')
    for c in ['Hawker Stall', 'Food court', 'Coffee shop', 'Cafe', 'Restaurant', 'Shopping mall']:
        s = df[df.category == c]
        print(f'  {c:14s} heat {s.heat.mean():5.1%} | aircon/cool {s.cool.mean():5.1%} | crowd {s.crowd.mean():5.1%} | n={len(s)}')

    print('\n=== Lab 5: where and when ===')
    df['month'] = df.date.dt.month
    m = df.groupby('month').heat.mean()
    print('heat complaints by month (%):', ' '.join(f'{int(k)}:{v*100:.1f}' for k, v in m.items()))
    cbd = (1.2840, 103.8510)  # Raffles Place
    km = np.hypot((df.lat - cbd[0]) * 111, (df.lon - cbd[1]) * 111)
    df['ring'] = pd.cut(km, [0, 3, 6, 10, 30], labels=['0-3 km', '3-6 km', '6-10 km', '10+ km'])
    print('by distance from the CBD:')
    print(df.groupby('ring', observed=True)[['heat', 'noise', 'crowd', 'quiet']].mean().mul(100).round(1).assign(n=df.groupby('ring', observed=True).size()).to_string())

    print('\n=== Lab 6: recommend comfortable places ===')
    g = df.groupby('place_id').agg(n=('text', 'size'), quiet=('quiet', 'sum'), noise=('noise', 'sum'),
                                   heat=('heat', 'sum'), cool=('cool', 'sum'), atm=('atmosphere', 'mean'))
    g = g.join(places.set_index('place_id')[['name', 'category', 'rating']])
    def show(t):
        t = t.assign(name=t.name.str.slice(0, 32))[['name', 'n', 'comfort', 'atm', 'rating']]
        print(t.round(2).to_string(index=False))
    k = 5  # shrinkage: as if every place had 5 extra neutral reviews
    g['comfort'] = ((g.quiet + g.cool) - (g.noise + g.heat)) / (g.n + k)
    want = g[g.category.isin(['Cafe', 'Coffee shop']) & (g.n >= 6)]
    print('top 5 cafes by comfort words (min 6 reviews in the sample):')
    show(want.sort_values('comfort', ascending=False).head(5))
    print('top 5 cafes by Google star rating:')
    show(want.sort_values(['rating', 'n'], ascending=False).head(5))
    print(f'correlation between comfort score and star rating across {len(want)} cafes: {want.comfort.corr(want.rating):.2f}')
