"""Rebuild the Aurora caption analysis from preserved source exports. Run: python analysis/analyze.py Classification is rule-assisted, with recorded contextual overrides. It concerns exported captions only, not unseen media, actual reach, or legal compliance. """ from pathlib import Path from datetime import datetime, timedelta, timezone from zoneinfo import ZoneInfo from collections import Counter, defaultdict import csv import hashlib import json import re import statistics from find_matches import normalize ROOT=Path(__file__).resolve().parent.parent OUT=ROOT/'analysis' SOURCES=['townofaurora','tom.mrakas','mrakas4aurora'] LABELS={'townofaurora':'Town of Aurora','tom.mrakas':'tom.mrakas','mrakas4aurora':'mrakas4aurora'} CATS={1:'Town related post',2:'Election related post',3:'Private content'} TZ=ZoneInfo('America/Toronto') VERSION='1.0.0' def dump(path,data): path.write_text(json.dumps(data,ensure_ascii=False,indent=2)+'\n') def classify(i,r): text=r['text']; t=normalize(text) info={'category_id':1,'category':CATS[1],'confidence':'high','needs_review':False, 'election_subtype':None,'rationale':'Civic, community, local business, public service, or mayoral-duty content; no substantive electoral appeal identified.', 'method':'full-caption rules with contextual excerpt review', 'version':VERSION, 'possible_campaign_branding':False,'incidental_election_mention':False} def assign(cat,why,confidence='high',sub=None): info.update(category_id=cat,category=CATS.get(cat),rationale=why,confidence=confidence,needs_review=confidence!='high',election_subtype=sub) # Captionless and contextless records are deliberately not assigned a fictional category. if not re.search(r'\w',t) or i in [108,199,728,948,1209]: assign(None,'No usable caption, or caption requires missing image/video/shared-post context. Category cannot be established from this export.','unassessable') return info private=[688,691,769,771,780,841,919,939,963,975,984,989,995,1010,1032,1082,1107,1124,1143,1174,1222,1272,1347,1386,1409,1419,1422,1581,1587,1598,1599,1610,1635,1690] if i in private: assign(3,'Personal family milestone, remembrance, hobby, leisure, or non-civic personal experience. Local references alone do not make the post municipal content.', 'medium' if i in [780,841,939,963,975,1032,1107,1124,1222,1272,1347,1409,1422,1598] else 'high') return info # Common false positives: a clerk's job description is not an election communication. if i in [597,1330]: info.update(incidental_election_mention=True,rationale='Town Clerk recruitment/appointment. Election administration is mentioned only as a job duty; primary purpose is municipal staffing.') return info official=all(m['source']=='townofaurora' for m in r['members']) electoral=re.search(r'\belections?\b|\bre[- ]?elect(?:ion)?\b|\bvoters?\b|\bballot\b|\bcanvass\w*|\bnomination day\b|\ball[- ]candidates?\b|aurora\.ca/(?:candidates|election|voters)|#reelect',t) campaign=re.search(r'endorsement alert|video endorsement|my campaign|our (?:re-election )?campaign|the campaign (?:has begun|in high gear)|team mrakas|i support mayor mrakas|show your support with a lawn sign',t) overrides={695:'Campaign canvassing with Team Mrakas volunteers.',701:'Compares the author with named political opponents and asks residents to judge competing approaches.', 706:'Explicit contrast between election-cycle promises (every four years) and the author’s leadership.', 708:'Explicit reference to the active campaign, embedded in a community-market update.', 710:'Campaign lawn-sign demand and distribution.',714:'Candidate questionnaire response; invites readers to compare all candidates’ answers.', 720:'Team Mrakas campaign lawn-sign distribution.',722:'Campaign sign delivery and sign requests.', 800:'Flyer delivery promoted during the campaign; electoral purpose inferred from campaign context.', 818:'Explicit campaign launch and door-to-door activity.',829:'Door-to-door activity in campaign context; mixed with a market visit.', 830:'Campaign lawn-sign requests.',897:'Explicit invitation to support the author through campaign lawn signs.', 931:'Substantive discussion of allegations concerning campaign resources; allegations are reported as the author’s claims, not independently verified.', 1024:'Campaign booth, support stickers, and campaign volunteers at a street festival.'} if electoral or campaign or i in overrides: sub='neutral_election_information' if official or i in [696,712,857] else 'campaign_promotion_or_commentary' assign(2,overrides.get(i,'Substantive election participation, candidate/ballot information, endorsement, re-election, or campaign wording.'), 'medium' if i in [701,706,800,829] else 'high',sub) return info # Promotional wording is retained as a separate sensitivity flag, not presumed campaigning. possible=[683,732,735,739,803,869,916,1175,1216,1239,1257,1344,1399,1408,1412,1418,1489,1650,1652,1664,1673] if i in possible: info.update(possible_campaign_branding=True,confidence='medium',needs_review=True, rationale='Civic achievement, leadership promotion, slogan, or newsletter appeal without an explicit electoral request. Conservative Town classification; review missing media for possible campaigning.') elif re.search(r'#leadershipthat',t): info['rationale']='Civic content with leadership branding. A leadership hashtag alone is not sufficient evidence of election campaigning.' if len(t)<75 or i in [1145,1287,1385,1640]: info.update(confidence='medium',needs_review=True) return info def build(): inventory=json.loads((OUT/'text_inventory.json').read_text()) original={s:json.loads((ROOT/'backups'/f'posts-{s}.json').read_text()) for s in SOURCES} posts=[]; by_inv=defaultdict(list); by_key={} overrides={} for i,r in enumerate(inventory): c=classify(i,r);overrides[r['key']]=c for m in r['members']: raw=original[m['source']][m['index']] assert raw['post_id']==m['id'] and raw['date']==m['date'], 'Inventory does not match the preserved source snapshot' assert normalize(raw['text'])==normalize(r['text']), 'Caption inventory mismatch' dt=datetime.fromisoformat(raw['date'].replace('Z','+00:00')) p={**raw,'source':m['source'],'source_index':m['index'],'key':m['source']+':'+raw['post_id'], 'inventory_index':i,'local_date':dt.astimezone(TZ).date().isoformat(), 'local_datetime':dt.astimezone(TZ).isoformat(), 'timestamp':dt.timestamp(),'category':c['category'],'category_id':c['category_id'],'classification':c, 'text_usable':bool(re.search(r'\w',raw['text']))} posts.append(p);by_inv[i].append(p);by_key[p['key']]=p dump(OUT/'classification_decisions.json',overrides) candidates=json.loads((OUT/'match_candidates.json').read_text()) # Each proposal was inspected using its caption context and token-level changes. reject={(482,489),(517,550)} accept={(1501,1683),(1343,1663),(1308,1659),(1179,1645),(34,178),(105,797),(1369,1666),(20,191),(1082,1635),(1584,1692)} partial={(1548,1688),(964,1627),(469,1353),(74,760)} # Shared-event wording with large rewrites is left out of near-copy totals. links=[] for c in candidates: pair=(c['a'],c['b']) kind='near' if ((c['automatic_proposal'] and pair not in reject) or pair in accept) else 'partial' if pair in partial else 'related_only' reason=('Reviewed minor wording/formatting variant of the same caption.' if kind=='near' else 'Substantial shared passage or shortened caption; kept separate from near-copy headline totals.' if kind=='partial' else 'Different dates, numbers, location/status, recurring-event template, or insufficient common text; excluded from duplicate totals.') discrepancy = 'Same-day, otherwise matching State of the Town captions refer to different years: 2026 versus 2025.' if pair==(1308,1659) else None if discrepancy:reason='Reviewed near copy despite a conflicting year; both source timestamps are February 18, 2026. Preserve the discrepancy for review.' links.append({**c,'kind':kind,'decision_reason':reason,'factual_discrepancy':discrepancy}) dump(OUT/'match_decisions.json',links) exact_groups=json.loads((OUT/'exact_groups.json').read_text()) # Sets for each inventory record. Near/partial group merging requires complete linkage, # so A~B and B~C cannot silently imply A~C. modes={};groups_by_mode={} for mode in ['exact','near','partial']: allowed={'exact'} | ({'near'} if mode in ['near','partial'] else set()) | ({'partial'} if mode=='partial' else set()) sets=[set(g) for g in exact_groups] approved={frozenset((c['a'],c['b'])) for c in links if c['kind'] in allowed} owner={i:g for g,ids in enumerate(sets) for i in ids} exact_rep={i:ids[0] for ids in exact_groups for i in ids} for c in links: if c['kind'] not in allowed:continue ga,gb=owner[c['a']],owner[c['b']] if ga==gb:continue if all(exact_rep[a]==exact_rep[b] or frozenset((exact_rep[a],exact_rep[b])) in approved for a in sets[ga] for b in sets[gb]): for i in sets[gb]:owner[i]=ga sets[ga]|=sets[gb];sets[gb]=set() groups=[] for ids in sets: if not ids:continue members=sorted([p for i in ids for p in by_inv[i]],key=lambda p:(p['timestamp'],p['key'])) gid=mode+'-'+hashlib.sha256('|'.join(sorted(p['key'] for p in members)).encode()).hexdigest()[:10] sources=sorted(set(p['source'] for p in members),key=SOURCES.index) usable=all(p['text_usable'] for p in members) group={'id':gid,'members':[p['key'] for p in members],'sources':sources,'size':len(members),'usable':usable, 'short_caption':max(len(p['text'].strip()) for p in members)<50,'variant_count':len(ids), 'first_date':members[0]['local_date'],'first_source':members[0]['source']} groups.append(group) for p in members: count=sum(q['source']==p['source'] for q in members) p.setdefault('duplicates',{})[mode]={'group':gid,'group_size':len(members),'sources':sources, 'cross_source':usable and len(sources)>1,'within_source':usable and count>1, 'repeat_after_first':usable and p is not members[0], 'source_first':p is next(q for q in members if q['source']==p['source']), 'has_any_match':usable and len(members)>1,'short_caption':group['short_caption']} groups_by_mode[mode]=groups summary=[] for s in SOURCES: ps=[p for p in posts if p['source']==s]; gs=[g for g in groups if s in g['sources'] and g['usable']] cats=Counter(p['category_id'] for p in ps) election=sorted([p for p in ps if p['category_id']==2],key=lambda p:p['timestamp']) campaign=[p for p in election if p['classification']['election_subtype']=='campaign_promotion_or_commentary'] summary.append({'source':s,'label':LABELS[s],'total':len(ps),'start':min(p['local_date'] for p in ps),'end':max(p['local_date'] for p in ps), 'categories':{str(k):cats[k] for k in [1,2,3,None]},'election':len(election),'campaign':len(campaign), 'neutral_election':len(election)-len(campaign),'first_election':election[0]['key'] if election else None, 'first_campaign':campaign[0]['key'] if campaign else None, 'distinct_captions':len(gs),'exclusive_groups':sum(g['sources']==[s] for g in gs), 'exclusive_posts':sum(not p['duplicates'][mode]['cross_source'] and p['text_usable'] for p in ps), 'singletons':sum(p['duplicates'][mode]['group_size']==1 and p['text_usable'] for p in ps), 'cross_posts':sum(p['duplicates'][mode]['cross_source'] for p in ps), 'within_posts':sum(p['duplicates'][mode]['within_source'] for p in ps), 'within_extra':sum(p['text_usable'] and not p['duplicates'][mode]['source_first'] for p in ps), 'missing_text':sum(not p['text'].strip() for p in ps),'unusable_text':sum(not p['text_usable'] for p in ps), 'review':sum(p['classification']['needs_review'] for p in ps),'possible_campaign':sum(p['classification']['possible_campaign_branding'] for p in ps)}) matrix=[] for a in SOURCES: for b in SOURCES: if a==b:continue pa=[p for p in posts if p['source']==a] ga={g['id'] for g in groups if a in g['sources'] and g['usable']} gb={g['id'] for g in groups if b in g['sources'] and g['usable']} matched=sum(p['text_usable'] and b in p['duplicates'][mode]['sources'] for p in pa) matrix.append({'from':a,'to':b,'matched':matched,'denominator':len(pa),'percent':round(matched/len(pa)*100,2), 'text_denominator':sum(p['text_usable'] for p in pa),'shared_groups':len(ga&gb), 'jaccard':round(len(ga&gb)/len(ga|gb)*100,2) if ga|gb else 0}) intersections=[] for mask in range(1,8): ss=[s for j,s in enumerate(SOURCES) if mask&(1<1 for g in gs),'repeated_posts':sum(g['size'] for g in gs if g['size']>1)}) # Lead/lag uses each source's earliest appearance of a shared caption group. timing=[] for j,a in enumerate(SOURCES): for b in SOURCES[j+1:]: lags=[] for g in groups: if a in g['sources'] and b in g['sources'] and g['usable']: ap=min(by_key[k]['timestamp'] for k in g['members'] if by_key[k]['source']==a) bp=min(by_key[k]['timestamp'] for k in g['members'] if by_key[k]['source']==b) lags.append((bp-ap)/60) timing.append({'a':a,'b':b,'shared':len(lags),'a_first':sum(x>0 for x in lags),'b_first':sum(x<0 for x in lags),'ties':sum(x==0 for x in lags), 'within_10_minutes':sum(abs(x)<=10 for x in lags),'median_absolute_minutes':round(statistics.median(map(abs,lags)),2) if lags else None}) mayor_town_content=[] for s in SOURCES[1:]+['combined']: ps=[p for p in posts if p['source'] in SOURCES[1:] and (s=='combined' or p['source']==s)] civic=[p for p in ps if p['category_id']==1] absent=[p for p in civic if p['text_usable'] and 'townofaurora' not in p['duplicates'][mode]['sources']] mayor_town_content.append({'source':s,'total_posts':len(ps),'town_related_posts':len(civic), 'town_related_without_town_match':len(absent), 'percent_of_all_posts':round(100*len(absent)/len(ps),2) if ps else None, 'percent_of_town_related_posts':round(100*len(absent)/len(civic),2) if civic else None}) modes[mode]={'summary':summary,'matrix':matrix,'intersections':intersections,'timing':timing, 'mayor_town_content':mayor_town_content, 'distinct_usable_groups':sum(g['usable'] for g in groups),'unassessable_records':sum(not p['text_usable'] for p in posts)} posts.sort(key=lambda p:(p['timestamp'],p['key'])) start=min(p['local_date'] for p in posts);end=max(p['local_date'] for p in posts) # Exact-hour overlap and day-trimmed overlap identify collection boundary effects. overlap_start=max(min(p['timestamp'] for p in posts if p['source']==s) for s in SOURCES) overlap_end=min(max(p['timestamp'] for p in posts if p['source']==s) for s in SOURCES) quality={'empty_text':sum(not p['text'].strip() for p in posts),'unusable_text':sum(not p['text_usable'] for p in posts), 'duplicate_source_ids':sum(len(v)-len(set(p['post_id'] for p in v)) for v in original.values()), 'timestamp_mismatches':sum(abs(p['timestamp']-p['creation_time'])>1 for p in posts), 'outside_common_hours':sum(not overlap_start<=p['timestamp']<=overlap_end for p in posts), 'common_start_utc':datetime.fromtimestamp(overlap_start,timezone.utc).isoformat(), 'common_end_utc':datetime.fromtimestamp(overlap_end,timezone.utc).isoformat()} result={'version':VERSION,'generated_at':datetime.now(timezone.utc).isoformat(),'timezone':str(TZ),'start':start,'end':end, 'sources':SOURCES,'labels':LABELS,'total_posts':len(posts),'quality':quality,'modes':modes,'posts':posts,'groups':groups_by_mode, 'match_decisions':links,'source_sha256':{s:hashlib.sha256((ROOT/'backups'/f'posts-{s}.json').read_bytes()).hexdigest() for s in SOURCES}} dump(OUT/'results.json',result) for s,rows in original.items(): updated=[] for idx,row in enumerate(rows): p=by_key[s+':'+row['post_id']] updated.append({**row,'category_id':p['category_id'],'category':p['category'], 'analysis':{'version':VERSION,'timezone':str(TZ),'local_date':p['local_date'], 'classification':p['classification'],'duplicates':p['duplicates']}}) dump(ROOT/f'posts-{s}.json',updated) fields=['source','post_id','date','local_date','category_id','category','confidence','needs_review','election_subtype','possible_campaign_branding','duplicate_group','cross_source_duplicate','within_source_duplicate','post_url','text','rationale'] with (OUT/'posts.csv').open('w',newline='') as f: writer=csv.DictWriter(f,fieldnames=fields);writer.writeheader() for p in posts: c=p['classification'];d=p['duplicates']['near'] row={k:p.get(k,'') for k in fields};row.update({k:c[k] for k in ['confidence','needs_review','election_subtype','possible_campaign_branding','rationale']}) row.update(duplicate_group=d['group'],cross_source_duplicate=d['cross_source'],within_source_duplicate=d['within_source']) # Prevent spreadsheet formula interpretation of arbitrary exported captions. for k,v in row.items(): if isinstance(v,str) and v.startswith(('=','+','-','@')):row[k]="'"+v writer.writerow(row) with (OUT/'daily_counts.csv').open('w',newline='') as f: fields=['date','source','posts','town','election','private','unclassified','exact_cross_source','near_cross_source','within_source','extra_repeated_occurrences'] w=csv.DictWriter(f,fieldnames=fields);w.writeheader() buckets=defaultdict(list) for p in posts:buckets[p['local_date'],p['source']].append(p) day=datetime.fromisoformat(start).date();last=datetime.fromisoformat(end).date() while day<=last: for s in SOURCES: ps=buckets[day.isoformat(),s] w.writerow({'date':day.isoformat(),'source':s,'posts':len(ps),**{k:sum(p['category_id']==cat for p in ps) for k,cat in [('town',1),('election',2),('private',3),('unclassified',None)]}, 'exact_cross_source':sum(p['duplicates']['exact']['cross_source'] for p in ps), 'near_cross_source':sum(p['duplicates']['near']['cross_source'] for p in ps), 'within_source':sum(p['duplicates']['near']['within_source'] for p in ps), 'extra_repeated_occurrences':sum(p['duplicates']['near']['repeat_after_first'] for p in ps)}) day+=timedelta(days=1) return result if __name__=='__main__': result=build() print(json.dumps({'summary':result['modes']['near']['summary'],'matrix':result['modes']['near']['matrix'],'quality':result['quality']},indent=2))