"""Published with the Unicorn Hunter article "What Jobs Are AI Startups Hiring For? November 2026 Data".
The input paths below point to our working copy of the November 3, 2026 snapshot. postings.csv in this
folder holds the same postings after deduplication, with the labels this script assigns."""
#!/usr/bin/env python3
"""Reproduce Unicorn Hunter's supplied public-board snapshot; Python stdlib only.
Run: python3 /home/ubuntu/ref/blog2/data/analysis.py
The 60-row human audit is stored in spotcheck_labels.json beside this file.
No job-description eligibility is inferred from title keywords.
"""
import json, re, random, statistics, hashlib
from pathlib import Path
from collections import Counter
from datetime import datetime, date
ROOT = Path('/home/ubuntu/ref/blog2/data')
JOBS = Path('/home/ubuntu/ref/realjobs/jobs_2026-11-03.json')
BOARDS = Path('/home/ubuntu/ref/realjobs/boards.json')
COMPANIES = Path('/home/ubuntu/unihunt/client/src/content/companies.ts')
SNAPSHOT = date(2026,11,3)
SEED = 20261103
FUNCTIONS = ['Engineering','Research','Product','Design','Data Science & Analytics','Sales & GTM','Marketing','Operations & Strategy','Finance','Legal & Policy','Other']
# First matching title pattern wins; specific business functions precede
# engineer/research keywords to avoid Finance Engineer/Research Operations errors.
TITLE_RULES = [
 ('Finance',r'finance|financial (?!services)|accounting|accountant|payroll|treasury|tax\b|billing operations|procurement|controller|cfo\b'),
 ('Legal & Policy',r'legal|counsel|paralegal|policy|government affairs|public affairs|litigation|compliance|governance'),
 ('Operations & Strategy',r'operations|\bops\b|bizops|revops|chief of staff|recruit|talent|people\b|human resources|workplace|office (?:manager|coordinator)|executive assistant|administrative|program manag|\btpm\b|project manag|deployment strategist|transformation|strategy|strategic projects|business analyst|sourcing|benefits|supply chain|logistics|onboarding coordinator|corporate development'),
 ('Design',r'designer|design engineer|product design|design lead|head of design|design director|member of design staff|creative|cinematograph|video|filmmak|animation|motion design'),
 ('Engineering',r'partner deployed engineer|salesforce developer|tech lead manager'),
 ('Marketing',r'market(?:ing|er)|brand|communications|public relations|\bpr\b|community|content|copywriter|demand gen|paid media|analyst relations|developer (?:advocate|relations)|growth manager|growth lead'),
 ('Sales & GTM',r'\bsales\b|account executive|account manag|\bsdr\b|\bbdr\b|business development|solutions|solution architect|customer\s+success|client success|ai success|customer experience|customer support|technical support|support (?:engineer|specialist)|partner|commercial|engagement manager|gtm|go.to.market|applied ai architect|agent product manager|customer engineer|agent architect|implementation manager'),
 ('Data Science & Analytics',r'data scien|data analy|analytics|business intelligence|\bbi analyst\b|data quality|data annot|data label'),
 ('Engineering',r'engineer|developer|\bsre\b|\bit\b|security|infrastructure|technical staff|\bmts\b|technician|technical lead|\bciso\b|machine learning'),
 ('Research',r'research|scientist|research resident|fellows program'),
 ('Product',r'product manager|product lead|head of product|product director|product management|product owner'),
]
DEPT_RULES = [
 ('Finance',r'finance'), ('Legal & Policy',r'legal|policy|governance|corporate affairs'),
 ('Marketing',r'marketing|communications|growth|\bpr\b'),
 ('Operations & Strategy',r'operations|people|recruiting|talent|human resources|workplace|founders office|transformation|deployment strategists'),
 ('Sales & GTM',r'sales|gtm|go to market|revenue|customer|commercial|partnership|business development|solutions'),
 ('Design',r'design'), ('Data Science & Analytics',r'data science|analytics|data quality'),
 ('Research',r'research|modeling'), ('Engineering',r'engineering|infra|platform|security|\bit\b|search|mobile|inference|machine learning|epd|builder'),
 ('Product',r'product'),
]
def classify(row, version='final'):
    title=row['title'].lower(); dept=(row.get('dept') or '').lower().strip()
    # Version initial intentionally retained to quantify corrections from review.
    initial=json.loads((ROOT/'initial_rules.json').read_text()) if version=='initial' else None
    rules=initial['title'] if initial else TITLE_RULES
    for function,pattern in rules:
        if re.search(pattern,title): return function,'title'
    for function,pattern in (initial['dept'] if initial else DEPT_RULES):
        if re.search(pattern,dept): return function,'department'
    return 'Other','unmatched'
EARLY = r'\bintern(?:ship)?\b|new[ -]?grad(?:uate)?|early[ -]career|entry[ -]level|\bjunior\b|\bjr\.?\b|university grad|co[ -]?op'
def seniority(title):
    t=re.sub(r'member of (?:technical|design|product|research|strategic) staff','member',title.lower())
    if re.search(EARLY,t): return 'Intern/new grad'
    # Chief of Staff is a function, not automatically a C-suite role.
    if re.search(r'\bdirector\b|\bhead\b|\bvp\b|vice president|president|\bceo\b|\bcfo\b|\bcto\b|\bciso\b|chief (?!of staff)',t): return 'Director/head and above'
    if re.search(r'\bmanager\b|\blead\b',t): return 'Manager/lead'
    if re.search(r'(?<!technical )\bstaff\b|\bprincipal\b|\bdistinguished\b',t) and 'chief of staff' not in t: return 'Staff/principal'
    if re.search(r'\bsenior\b|\bsr\.?\b',t): return 'Senior'
    return 'No level word'
CITY_PATTERNS = {
 'San Francisco':r'san francisco|\bsf office\b|mercor hq',
 'New York':r'new york|\bnyc\b', 'London':r'\blondon\b',
 'Seattle':r'\bseattle\b', 'Toronto':r'\btoronto\b',
 'Washington, DC':r'washington[, ]+(?:d\.?c\.?)', 'Mountain View':r'mountain view',
 'Montreal':r'montreal|montréal', 'Austin':r'\baustin\b', 'Boston':r'\bboston\b',
 'Palo Alto':r'palo alto','Foster City':r'foster city','Chicago':r'chicago|chi office',
 'Pittsburgh':r'pittsburgh|pgh office','San Francisco Bay Area (unspecified)':r'san francisco bay area',
}
US = r'san francisco|new york|\bnyc\b|sf office|mercor hq|seattle|washington|mountain view|foster city|palo alto|austin|boston|chicago|chi office|pgh office|pittsburgh|nashville|dallas|salt lake|houston|san antonio|los angeles|phoenix|richmond|denver|miami|new jersey|california|utah|united states|\bus\b'
CA = r'\bcanada\b|\bcan\b|toronto|ontario|montreal|montréal|ottawa|vancouver|calgary|halifax|winnipeg|edmonton|victoria'
NON_US_CA = r'london|united kingdom|\buk\b|europe|dublin|belgrade|berlin|munich|germany|france|paris|singapore|tokyo|japan|seoul|korea|sydney|melbourne|australia|india|bangalore|bengaluru|mumbai|madrid|spain|milan|italy|poland|bulgaria|sofia|brazil|sao paulo|são paulo|mexico|argentina|chile|colombia|sweden|stockholm|denmark|portugal|netherlands|amsterdam|belgium|brussels|switzerland|zurich|zürich|\bch\b|riyadh|saudi|dubai|doha|uae|united arab|middle east|austria|hungary|croatia|ireland|czechia|frankfurt|edinburgh'
def loc_group(location):
    l=location.lower().strip(); u=bool(re.search(US,l)); c=bool(re.search(CA,l)); o=bool(re.search(NON_US_CA,l)); remote='remote' in l
    # Strict location-only split. Explicit remote (including remote-friendly)
    # with US/Canada or North America qualifies, even if foreign options coexist.
    if remote and (u or c or 'north america' in l): return 'Remote North America'
    if u and not c and not o: return 'United States'
    if c and not u and not o: return 'Canada'
    if re.fullmatch(r'mexico(?: city)?(?:[, |]+mexico(?: city)?)*',l): return 'Other North America (Mexico)'
    if o and not u and not c and 'mexico' not in l: return 'Outside North America'
    if o and not u and not c: return 'Mixed outside US/Canada options'
    if u or c: return 'Mixed US/Canada or international options'
    return 'Unspecified geography'
def pct(n,d): return round(n/d*100,1) if d else 0.0
def main():
    ROOT.mkdir(parents=True,exist_ok=True)
    raw=json.loads(JOBS.read_text()); boards=json.loads(BOARDS.read_text()); unique=[]; seen=set(); duplicates=[]
    for row in raw:
        key=(row['company'],row['title'],row['location'])
        if key in seen: duplicates.append(row)
        else: seen.add(key); unique.append(row)
    companies={}
    pattern=r'(\w+): \{ name: "([^"]+)", round: "([^"]+)", roundDate: "([^"]+)",.*?source: "([^"]+)" \}'
    for cid,name,round_,round_date,source in re.findall(pattern,COMPANIES.read_text()):
        companies[cid]={'company':name,'latest_recorded_series_round':round_,'round_date':round_date,'round_source':source}
    sample=random.Random(SEED).sample(range(len(unique)),60)
    sample_rows=[{'index':i,'company':unique[i]['company'],'title':unique[i]['title'],'dept':unique[i].get('dept',''),'initial':classify(unique[i],'initial')[0],'final':classify(unique[i])[0]} for i in sample]
    (ROOT/'spotcheck_sample.json').write_text(json.dumps(sample_rows,indent=2,ensure_ascii=False)+'\n')
    labels_path=ROOT/'spotcheck_labels.json'
    labels=json.loads(labels_path.read_text()) if labels_path.exists() else {}
    matched_initial=sum(classify(unique[i],'initial')[0]==labels.get(str(i)) for i in sample)
    matched_final=sum(classify(unique[i])[0]==labels.get(str(i)) for i in sample)
    # jobs.ashbyhq.com/runway identifies CFO.ai. The requested video-AI
    # company uses runway-ml. Preserve raw rows, quarantine rather than replace.
    quarantined=[row for row in unique if row['company']=='runway']
    included=[row for row in unique if row['company']!='runway']
    classified=[dict(row,function=classify(row)[0],signal=classify(row)[1],seniority=seniority(row['title']),location_group=loc_group(row['location'])) for row in included]
    total=len(classified); counts=Counter(r['function'] for r in classified); cids=sorted({r['company'] for r in classified})
    per_company=[]
    for cid in cids:
        rows=[r for r in classified if r['company']==cid]; fc=Counter(r['function'] for r in rows); board=boards[cid][0]
        boardurl=('https://job-boards.greenhouse.io/' if board[0]=='greenhouse' else 'https://jobs.ashbyhq.com/')+board[1]
        per_company.append(dict(companies[cid],id=cid,postings=len(rows),share=pct(len(rows),total),top_function=fc.most_common(1)[0][0],function_counts=dict(fc),board=boardurl,ats=board[0]))
    function_rows=[{'function':f,'postings':counts[f],'share':pct(counts[f],total),'median_company_share':round(statistics.median([c['function_counts'].get(f,0)/c['postings']*100 for c in per_company]),1)} for f in FUNCTIONS]
    city_counts=Counter()
    for r in classified:
        for city,p in CITY_PATTERNS.items():
            if city=='San Francisco' and 'san francisco bay area' in r['location'].lower(): continue
            if re.search(p,r['location'].lower()): city_counts[city]+=1
    location_counts=Counter(r['location_group'] for r in classified); seniority_counts=Counter(r['seniority'] for r in classified)
    recency=Counter(); ages=[]; missing=future=0
    for r in classified:
        try: published=datetime.fromisoformat(r['published'].replace('Z','+00:00')).date()
        except (ValueError,TypeError,AttributeError): missing+=1; recency['Missing/invalid']+=1; continue
        age=(SNAPSHOT-published).days
        if age<0: future+=1; recency['Future-dated']+=1; continue
        ages.append(age)
        recency['0–7 days' if age<=7 else '8–30 days' if age<=30 else '31–90 days' if age<=90 else '91–180 days' if age<=180 else '181–365 days' if age<=365 else 'Over 365 days']+=1
    tech=counts['Engineering']+counts['Research']; outside_engineering=total-counts['Engineering']; early=seniority_counts['Intern/new grad']
    result={
      'snapshot_date':SNAPSHOT.isoformat(),'snapshot_date_long':'November 3, 2026','input_path':str(JOBS),'input_sha256':hashlib.sha256(JOBS.read_bytes()).hexdigest(),'script_path':str(Path(__file__).resolve()),
      'raw_postings':len(raw),'deduplicated_postings':len(unique),'analyzed_postings':total,'duplicates_removed':len(duplicates),'supplied_company_labels':len({r['company'] for r in raw}),'company_count':len(cids),'ats_company_counts':dict(Counter(c['ats'] for c in per_company)),
      'quarantine':{'postings':len(quarantined),'rows':quarantined,'reason':'Supplied runway board belongs to CFO.ai, not video-AI Runway','mismatched_board':'https://jobs.ashbyhq.com/runway','official_video_ai_careers':'https://runway.com/careers','correct_video_ai_board':'https://jobs.ashbyhq.com/runway-ml','recorded_funding':companies['runway']},
      'excluded_no_board':['Hebbia'],'function_rules':{'title_first_match':TITLE_RULES,'dept_fallback':DEPT_RULES,'research_engineers':'Engineering unless no engineering signal; research scientists Research','non_engineering_is_not_nontechnical':True},
      'audit':{'seed':SEED,'sample_size':len(sample),'labels_reviewed':len(labels),'initial_correct':matched_initial,'initial_accuracy':pct(matched_initial,len(sample)),'final_correct':matched_final,'final_accuracy':pct(matched_final,len(sample)),'same_sample_rechecked_not_holdout':True,'sample':sample_rows,'labels_path':str(labels_path)},
      'functions':function_rows,'per_company':sorted(per_company,key=lambda c:-c['postings']),
      'locations':[{'location':k,'postings':v,'share':pct(v,total)} for k,v in location_counts.items()],
      'location_rules':{'exclusive_precedence':['explicit remote+US/Canada/NA','US only','Canada only','Mexico only','outside North America only','mixed options','unspecified'],'Mexico':'Separate geographic North America bucket; mixed Mexico/other countries separate','remote_friendly_is_not_guaranteed_remote':True},
      'cities_top8':[{'city':k,'postings_mentioning_city':v,'share':pct(v,total)} for k,v in city_counts.most_common(8)],
      'seniority':[{'level':k,'postings':seniority_counts[k],'share':pct(seniority_counts[k],total)} for k in ['Intern/new grad','No level word','Senior','Staff/principal','Manager/lead','Director/head and above']],
      'seniority_rules':{'early_pattern':EARLY,'precedence':['early keywords','director/head/VP/C-suite','manager/lead','staff/principal','senior','no level word'],'technical_staff_not_seniority':True,'manager_words_not_proof_of_people_management':True},
      'early_career':{'explicit_title_signal_postings':early,'explicit_title_signal_share':pct(early,total),'confirmed_eligible_share':None,'eligibility_limit':'Descriptions absent; no level word does not imply entry level; internship may require graduate education.'},
      'recency':[{'age':k,'postings':recency[k],'share':pct(recency[k],total)} for k in ['0–7 days','8–30 days','31–90 days','91–180 days','181–365 days','Over 365 days','Missing/invalid','Future-dated']],
      'recency_summary':{'median_calendar_days':statistics.median(ages),'within_30_days':sum(0<=a<=30 for a in ages),'within_30_days_share':pct(sum(0<=a<=30 for a in ages),total),'within_90_days':sum(0<=a<=90 for a in ages),'within_90_days_share':pct(sum(0<=a<=90 for a in ages),total),'missing':missing,'future_dated':future,'date_semantics':'supplied published field; some ATS fields represent last update rather than initial posting; calendar dates rather than exact elapsed hours'},
      'findings':{'outside_engineering_postings':outside_engineering,'outside_engineering_share':pct(outside_engineering,total),'outside_engineering_and_research_postings':total-tech,'outside_engineering_and_research_share':pct(total-tech,total),'sales_plus_marketing':counts['Sales & GTM']+counts['Marketing'],'sales_plus_marketing_share':pct(counts['Sales & GTM']+counts['Marketing'],total),'anthropic_share':next(c['share'] for c in per_company if c['id']=='anthropic')},
      'duplicate_rows':duplicates,'classification_signal_counts':dict(Counter(r['signal'] for r in classified))
    }
    (ROOT/'results.json').write_text(json.dumps(result,indent=2,ensure_ascii=False)+'\n')
    (ROOT/'classified_jobs.json').write_text(json.dumps(classified,indent=2,ensure_ascii=False)+'\n')
    print(json.dumps({k:result[k] for k in ['raw_postings','deduplicated_postings','analyzed_postings','duplicates_removed','functions','locations','cities_top8','seniority','recency','recency_summary','findings']},indent=2,ensure_ascii=False))
    print('\nAUDIT SAMPLE:')
    for x in sample_rows: print(f"{x['index']} | {x['company']} | {x['title']} | dept={x['dept']} | {x['initial']} -> {x['final']}")
if __name__=='__main__': main()