import fs from 'fs'
const all = JSON.parse(fs.readFileSync(process.argv[2], 'utf8'))
const norm = s => s.toLowerCase().replace(/[^a-z0-9 ]/g, ' ').replace(/\s+/g, ' ').trim()

// "Customer Success Engineer", "Customer Success Operations Analyst" and "Strategy & Operations,
// Customer Success" match the include pattern and are engineering, analytics or planning jobs, not
// the account-owning role. Architect, business partner and executive titles are kept: they own accounts.
const INCLUDE = /\b(customer|client) success\b/i
const EXCLUDE = /\b(engineer|administrator|analyst|operations|ops|strategist|strategy|program manager|product marketing|applications|services|forward deployed|enablement)\b/i

const seen = new Set(), jobs = []
for (const j of all.filter(x => INCLUDE.test(x.title) && !EXCLUDE.test(x.title))) {
  const k = j.co + '|' + norm(j.title) + '|' + j.content.length
  if (!seen.has(k)) { seen.add(k); jobs.push(j) }
}
const seenEx = new Set(); let excluded = 0
for (const j of all.filter(x => INCLUDE.test(x.title) && EXCLUDE.test(x.title))) {
  const k = j.co + '|' + norm(j.title) + '|' + j.content.length
  if (!seenEx.has(k)) { seenEx.add(k); excluded++ }
}
const unesc = s => s.replace(/&lt;/g,'<').replace(/&gt;/g,'>').replace(/&amp;/g,'&')
  .replace(/&quot;/g,'"').replace(/&#39;/g,"'").replace(/&nbsp;/g,' ')
const texts = jobs.map(j => unesc(j.content).replace(/<[^>]+>/g,' ').replace(/\s+/g,' '))

// Benefits boilerplate ("fertility and adoption assistance") is not product adoption. It
// appears in a handful of postings from one employer; the term is tested on text with it removed.
const BENEFIT_ADOPTION = /\b(fertility|infertility|surrogacy)\b[^.]{0,40}\badoption\b|\badoption (assistance|leave|benefits?|reimbursement|and surrogacy)\b/gi

const T = [
 ['Retention & revenue','renewals',/\brenewals?\b|\brenewing\b/i],['Retention & revenue','churn',/\bchurn/i],
 ['Retention & revenue','upsell / cross-sell',/\bupsell|\bcross-?sell/i],['Retention & revenue','expansion',/\bexpansion\b|\bexpand(ing)? (the |their |our )?(accounts?|footprint|usage|revenue|relationships?)/i],
 ['Retention & revenue','net revenue retention',/\bNRR\b|\bnet revenue retention\b|\bnet dollar retention\b|\bNDR\b/],
 ['Retention & revenue','gross retention',/\bGRR\b|\bgross (revenue |dollar )?retention\b/],
 ['Retention & revenue','ARR / MRR',/\bARR\b|\bMRR\b|\bannual recurring revenue\b/],
 ['Retention & revenue','quota / revenue target',/\bquota\b|\brevenue targets?\b|\bretention targets?\b|\bbookings\b/i],
 ['Retention & revenue','forecasting',/\bforecast/i],
 ['Account management','onboarding',/\bonboarding\b/i],['Account management','adoption',/\badoption\b/i,t => t.replace(BENEFIT_ADOPTION, ' ')],
 ['Account management','health score',/\bhealth (score|scoring|scores|metrics?)\b/i],
 ['Account management','QBR / EBR',/\bQBRs?\b|\bEBRs?\b|\b(quarterly|executive) business reviews?\b/i],
 ['Account management','time to value',/\btime[- ]to[- ]value\b|\bTTV\b/i],
 ['Account management','success plans',/\bsuccess plans?\b/i],
 ['Account management','escalation',/\bescalat/i],
 ['Account management','implementation',/\bimplementation\b/i],
 ['Account management','executive relationships',/\bC[- ]level\b|\bexecutive sponsors?\b|\bexecutive (relationships?|stakeholders?|engagement)\b|\bc-suite\b/i],
 ['Account management','book of business',/\bbook of business\b|\bportfolio of (accounts|customers|clients)\b/i],
 ['Account management','playbooks',/\bplaybooks?\b/i],
 ['Account management','voice of customer',/\bvoice of (the )?customer\b|\bVoC\b/],
 ['Account management','product feedback',/\bproduct (feedback|team|roadmap)\b|\bfeedback to (the )?product\b/i],
 ['Customer segment','enterprise',/\benterprise\b/i],['Customer segment','mid-market',/\bmid[- ]?market\b/i],
 ['Customer segment','SMB',/\bSMB\b|\bsmall (and|&) medium\b/i],
 ['Customer segment','SaaS',/\bSaaS\b|\bsoftware[- ]as[- ]a[- ]service\b/i],
 ['Customer segment','strategic accounts',/\bstrategic accounts?\b|\bkey accounts?\b/i],
 ['Tools','Salesforce',/\bSalesforce\b/i],['Tools','Gainsight',/\bGainsight\b/i],
 // Case-sensitive: "vitally important" is an adverb. "Catalyst" is a platform too, but General Catalyst is an
 // investor named in company blurbs, so it is left out rather than counted loosely.
 ['Tools','Totango / ChurnZero / Planhat / Vitally',/\bTotango\b|\bChurnZero\b|\bPlanhat\b|\bVitally\b/],
 ['Tools','HubSpot',/\bHubSpot\b/i],['Tools','Zendesk',/\bZendesk\b/i],
 ['Tools','Jira',/\bJira\b/i],['Tools','SQL',/\bSQL\b/],
 ['Tools','Looker / Tableau',/\bLooker\b|\bTableau\b/i],
 ['Tools','CRM (any)',/\bCRM\b/],['Tools','Excel / Sheets',/\bExcel\b|\bGoogle Sheets\b/i],
 ['Skills & background','cross-functional',/\bcross[- ]functional\b/i],
 ['Skills & background','technical aptitude',/\bAPIs?\b|\btechnical (background|aptitude|acumen|skills?)\b|\btechnically\b/i],
 ['Skills & background','project management',/\bproject management\b|\bPMP\b|\bproject plans?\b/i],
 ['Skills & background','language fluency',/\bbilingual\b|\bfluent in\b|\b(native|fluent) (speaker|level)\b|\bfluency\b/i],
 // "travel software" is Brex describing its own product, not a travel requirement.
 ['Skills & background','travel',/(?<!and )\btravel\b(?!\s+(software|and expense|&|management|perks|benefits|insurance|stipend|reimbursement))/i],
 ['Skills & background',"bachelor's degree",/\bbachelor/i],
]
const n = texts.length
const moe = p => +(1.96 * Math.sqrt((p/100)*(1-p/100)/n) * 100).toFixed(1)
const rows = T.map(([cat, term, re, prep]) => {
  const c = texts.filter(t => re.test(prep ? prep(t) : t)).length
  const pct = +(100*c/n).toFixed(1)
  return { cat, term, n: c, pct, moe: moe(pct) }
}).sort((a,b) => b.n - a.n)
// Postings from one company are not independent: Samsara alone supplies several postings that share
// boilerplate, so the usual binomial interval is too narrow. Everything below that compares groups, or
// ranks one term against another, resamples whole companies instead (cluster bootstrap, fixed seed so a
// re-run reproduces the file), and the group comparisons are then adjusted for the 43 terms compared
// (Benjamini-Hochberg, false discovery rate 5%).
const B = 4000
let seed = 20261113
const rand = () => { seed |= 0; seed = (seed + 0x6D2B79F5) | 0; let t = Math.imul(seed ^ (seed >>> 15), 1 | seed); t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; return ((t ^ (t >>> 14)) >>> 0) / 4294967296 }
const cos = [...new Set(jobs.map(j => j.co))]
const coIdx = new Map(cos.map((c, i) => [c, i]))
const hit = T.map(([, , re, prep]) => texts.map(t => re.test(prep ? prep(t) : t)))
const draws = Array.from({ length: B }, () => Array.from({ length: cos.length }, () => Math.floor(rand() * cos.length)))

// Per-company tallies for a set of jobs: how many postings, and how many name each term.
function tallies(include) {
  const nC = new Array(cos.length).fill(0)
  const hC = T.map(() => new Array(cos.length).fill(0))
  jobs.forEach((j, i) => {
    if (!include(j)) return
    const c = coIdx.get(j.co)
    nC[c]++
    T.forEach((_, t) => { if (hit[t][i]) hC[t][c]++ })
  })
  return { nC, hC }
}
const quantile = (sorted, q) => sorted[Math.min(sorted.length - 1, Math.max(0, Math.round(q * (sorted.length - 1))))]
const bhAdjust = ps => {
  const order = ps.map((p, i) => [p, i]).sort((x, y) => x[0] - y[0])
  const adj = new Array(ps.length)
  let min = 1
  for (let k = order.length - 1; k >= 0; k--) { min = Math.min(min, order[k][0] * order.length / (k + 1)); adj[order[k][1]] = min }
  return adj
}

// Share of one group's postings naming each term, for every bootstrap draw of companies.
function bootShares(tl) {
  return draws.map(d => {
    let n = 0; const h = new Array(T.length).fill(0)
    for (const c of d) { n += tl.nC[c]; for (let t = 0; t < T.length; t++) h[t] += tl.hC[t][c] }
    return h.map(x => (n ? x / n : NaN))
  })
}

function compare(inA, inScope = () => true) {
  const A = tallies(j => inScope(j) && inA(j)), Bt = tallies(j => inScope(j) && !inA(j))
  const nA = A.nC.reduce((x, y) => x + y, 0), nB = Bt.nC.reduce((x, y) => x + y, 0)
  const sa = bootShares(A), sb = bootShares(Bt)
  const base = T.map((_, t) => ({ a: A.hC[t].reduce((x, y) => x + y, 0), b: Bt.hC[t].reduce((x, y) => x + y, 0) }))
  const pvals = T.map((_, t) => {
    const d = sa.map((r, k) => r[t] - sb[k][t]).filter(x => !Number.isNaN(x)).sort((x, y) => x - y)
    const le = d.filter(x => x <= 0).length / d.length, ge = d.filter(x => x >= 0).length / d.length
    return { p: Math.max(1 / (d.length + 1), Math.min(1, 2 * Math.min(le, ge))), lo: quantile(d, 0.025), hi: quantile(d, 0.975) }
  })
  const adj = bhAdjust(pvals.map(x => x.p))
  const out = T.map(([cat, term], t) => ({
    cat, term,
    a: { n: base[t].a, pct: +(100 * base[t].a / nA).toFixed(1) },
    b: { n: base[t].b, pct: +(100 * base[t].b / nB).toFixed(1) },
    diff: +(100 * (base[t].a / nA - base[t].b / nB)).toFixed(1),
    lo: +(100 * pvals[t].lo).toFixed(1), hi: +(100 * pvals[t].hi).toFixed(1),
    q: +adj[t].toFixed(3), real: adj[t] < 0.05,
  }))
  return { a: nA, b: nB, rows: out }
}

// Titles are matched on keywords, so "Manager, Customer Success" (a team lead) is not senior here.
const SENIOR = /\b(senior|sr|principal|staff|lead|director|head of|vice president|vp|leader|chief)\b/i
const SENIOR_STRICT = /\b(senior|sr|principal|director|head of|vice president|vp|chief)\b/i
const LEADERSHIP = /\b(director|head of|vice president|vp|chief|leader|manager,? customer success|manager of)\b/i
const bySeniority = compare(j => SENIOR.test(j.title))
const byHost = compare(j => j.src === 'greenhouse')

// Does the seniority finding survive a different definition of senior?
const KEY = ['executive relationships', 'forecasting', 'Salesforce', 'cross-functional']
const seniorityChecks = [
  ['senior, principal, staff, lead, director, head, VP, chief or leader', SENIOR],
  ['senior, principal, director, head, VP or chief only (no lead or staff)', SENIOR_STRICT],
  ['the same plus team-lead titles such as "Manager, Customer Success"', new RegExp(SENIOR.source + '|' + LEADERSHIP.source, 'i')],
].map(([name, re]) => {
  const c = compare(j => re.test(j.title))
  return { name, senior: c.a, other: c.b, terms: KEY.map(k => { const r = c.rows.find(x => x.term === k); return { term: k, senior: r.a.pct, other: r.b.pct, diff: r.diff, real: r.real } }) }
})

// Is the host gap just a seniority gap? Compare the share of senior titles on each side, then redo the
// host comparison inside the non-senior titles only.
const seniorShare = src => {
  const js = jobs.filter(j => (src === 'greenhouse') === (j.src === 'greenhouse'))
  return { n: js.length, pct: +(100 * js.filter(j => SENIOR.test(j.title)).length / js.length).toFixed(1) }
}
const nonSeniorHost = compare(j => j.src === 'greenhouse', j => !SENIOR.test(j.title))
const HOST_KEY = ['QBR / EBR', 'implementation', 'Salesforce', 'executive relationships']
const hostChecks = {
  seniorShare: { greenhouse: seniorShare('greenhouse'), other: seniorShare('other') },
  nonSenior: { a: nonSeniorHost.a, b: nonSeniorHost.b, terms: HOST_KEY.map(k => nonSeniorHost.rows.find(x => x.term === k)).map(r => ({ term: r.term, greenhouse: r.a.pct, other: r.b.pct, diff: r.diff, lo: r.lo, hi: r.hi, real: r.real })) },
}

// Company-level widths for every term, and the head-to-head claims the page makes.
const whole = tallies(() => true)
const sh = bootShares(whole)
const widths = T.map((_, t) => {
  const v = sh.map(r => r[t]).sort((x, y) => x - y)
  return +(100 * (quantile(v, 0.975) - quantile(v, 0.025)) / 2).toFixed(1)
})
const termIdx = name => T.findIndex(x => x[1] === name)
const OUTCOME_TERMS = ['churn', 'net revenue retention', 'gross retention', 'ARR / MRR', 'quota / revenue target', 'health score', 'time to value', 'forecasting']
const pairShare = (x, y) => {
  const tx = termIdx(x), ty = termIdx(y)
  const d = sh.map(r => r[tx] - r[ty]).sort((p, q) => p - q)
  const base = (whole.hC[tx].reduce((p, q) => p + q, 0) - whole.hC[ty].reduce((p, q) => p + q, 0)) / n
  const lo = quantile(d, 0.025), hi = quantile(d, 0.975)
  return { a: x, b: y, diff: +(100 * base).toFixed(1), lo: +(100 * lo).toFixed(1), hi: +(100 * hi).toFixed(1), real: lo > 0 || hi < 0 }
}
const topOutcome = [...OUTCOME_TERMS].sort((x, y) => (whole.hC[termIdx(y)].reduce((p, q) => p + q, 0)) - (whole.hC[termIdx(x)].reduce((p, q) => p + q, 0)))[0]
const pairs = [
  pairShare('adoption', 'renewals'), pairShare('adoption', 'expansion'), pairShare('renewals', 'expansion'),
  ...['adoption', 'renewals', 'expansion', 'onboarding'].map(t => pairShare(t, topOutcome)),
  pairShare('Salesforce', 'Gainsight'),
]

// Lines quoted on the page. They were chosen by hand from sentences in the corpus; this step refuses to
// write the file unless each one appears word for word in a posting from that company, so a quote
// cannot be mistyped or outlive its source. The page says they are as collected on the study date.
const QUOTES = [
  { co: 'brex', name: 'Brex', terms: ['adoption'], text: 'Proactively manage an assigned book of accounts to drive customer adoption, value realization, and account growth.' },
  { co: 'samsara', name: 'Samsara', terms: ['adoption'], text: 'You pick up where our Implementation team leaves off, building the relationship, driving adoption, and ensuring customers realize measurable value from the product.' },
  { co: 'okta', name: 'Okta', terms: ['renewals', 'expansion'], text: 'Identify renewal risk and collaborate with internal teams to remediate, ensure a successful renewal, and proactively spot expansion opportunities.' },
  { co: 'pagerduty', name: 'PagerDuty', terms: ['renewals', 'expansion'], text: 'A track record of taking customers from little use to full use, and of renewal and expansion outcomes you can show.' },
  { co: 'gitlab', name: 'GitLab', terms: ['adoption', 'expansion'], text: 'Own a book of customers and drive adoption, retention, expansion, and overall satisfaction through ongoing strategic and technical engagement.' },
]
const quotes = QUOTES.map(q => {
  const found = jobs.some((j, i) => j.co === q.co && texts[i].includes(q.text))
  if (!found) throw new Error(`quote not found in any ${q.co} posting: ${q.text}`)
  return { company: q.name, terms: q.terms, text: q.text }
})

const perCompany = {}
for (const j of jobs) perCompany[j.co] = (perCompany[j.co] || 0) + 1
const bySource = {}
for (const src of ['greenhouse', 'lever', 'ashby']) {
  const js = jobs.filter(j => j.src === src)
  bySource[src] = { postings: js.length, companies: new Set(js.map(j => j.co)).size }
}
// `moe` is the company-resampled half-width. The binomial one, which treats every posting as independent,
// is kept as `moeIndependent` for comparison with the other studies.
rows.forEach(r => { r.moeIndependent = r.moe; r.moe = widths[termIdx(r.term)] })
const out = { role: 'Customer Success', n, companies: [...new Set(jobs.map(j=>j.co))].sort(),
  bySource, quotes, bySeniority, seniorityChecks, byHost, hostChecks, pairs, topOutcome, excluded, topCompanyShare: +(100 * Math.max(...Object.values(perCompany)) / n).toFixed(1),
  collected: new Date(Date.now() - new Date().getTimezoneOffset() * 60000).toISOString().slice(0,10),
  clusterBootstrap: { draws: B, unit: 'company', fdr: 0.05 },
  medianWords: texts.map(t=>t.split(' ').length).sort((a,b)=>a-b)[Math.floor(n/2)], rows }
if (process.argv[3]) fs.writeFileSync(process.argv[3], JSON.stringify(out, null, 1))
console.log(`n=${n} from ${out.companies.length} companies, median ${out.medianWords} words`)
rows.forEach((r,i)=>console.log(`  ${String(i+1).padStart(2)}. ${r.term.padEnd(30)} ${String(r.pct).padStart(5)}% ±${r.moe}  (${r.n})`))