import { BaseService, type BaseServiceContext } from '@learnitive/base-utils';
import { SERVICE_NAME } from '@learnitive/constants';
import { projectFilePath, type ProjectFilePath } from '@learnitive/project-storage';
import type { ProjectStorageService } from './project-storage-service';

const DEFAULT_MAX_DATASET_BYTES = 64 * 1024 * 1024;
const DEFAULT_SAMPLE_ROWS = 12;
const MAX_SAMPLE_ROWS = 50;
const MAX_COLUMNS = 512;
const QUANTILE_SAMPLE_LIMIT = 8192;
const CATEGORY_LIMIT = 4096;
const TOP_VALUE_LIMIT = 8;
const MAX_CELL_PREVIEW = 512;
const MISSING_VALUES = new Set(['', 'na', 'n/a', 'null', 'none', 'nan']);
const BOOLEAN_VALUES = new Set(['true', 'false', 'yes', 'no', 't', 'f']);
const ISO_DATE = /^\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?)?$/u;

export type DatasetFormat = 'csv' | 'tsv';
export type DatasetColumnType = 'empty' | 'integer' | 'number' | 'boolean' | 'date' | 'string';

export type DatasetTopValue = {
  value: string;
  count: number;
};

export type DatasetNumericStatistics = {
  count: number;
  min: number;
  max: number;
  mean: number;
  std: number;
  q1?: number;
  median?: number;
  q3?: number;
  quantilesApproximate: boolean;
};

export type DatasetColumnSummary = {
  index: number;
  name: string;
  inferredType: DatasetColumnType;
  missingCount: number;
  nonMissingCount: number;
  numeric?: DatasetNumericStatistics;
  uniqueCount?: number;
  uniqueCountCapped?: boolean;
  topValues?: readonly DatasetTopValue[];
};

export type DatasetInspectionOptions = {
  sampleRows?: number;
  statistics?: boolean;
  maxBytes?: number;
};

export type DatasetInspection = {
  path: ProjectFilePath;
  format: DatasetFormat;
  delimiter: ',' | '\t';
  bytes: number;
  rowCount: number;
  columnCount: number;
  columns: readonly DatasetColumnSummary[];
  sampleRows: readonly (readonly string[])[];
  malformedRowCount: number;
  warnings: readonly string[];
};

export type DatasetLatexTableOptions = {
  maxRows?: number;
  maxColumns?: number;
  caption?: string;
  label?: string;
  maxBytes?: number;
};

export type DatasetLatexTable = {
  path: ProjectFilePath;
  latex: string;
  rowCount: number;
  columnCount: number;
  includedRows: number;
  includedColumns: number;
  truncatedRows: boolean;
  truncatedColumns: boolean;
};

type ColumnAccumulator = {
  name: string;
  index: number;
  missingCount: number;
  nonMissingCount: number;
  couldBeInteger: boolean;
  couldBeNumber: boolean;
  couldBeBoolean: boolean;
  couldBeDate: boolean;
  numericCount: number;
  numericMean: number;
  numericM2: number;
  numericMin: number;
  numericMax: number;
  quantileSample: number[];
  categories: Map<string, number>;
  categoriesCapped: boolean;
};


function compactCell(value: string): string {
  if (value.length <= MAX_CELL_PREVIEW) return value;
  let hash = 2166136261;
  for (let index = 0; index < value.length; index += 1) {
    hash ^= value.charCodeAt(index);
    hash = Math.imul(hash, 16777619);
  }
  return `${value.slice(0, MAX_CELL_PREVIEW)}…[${value.length} chars; ${(hash >>> 0).toString(16)}]`;
}

function isMissing(value: string): boolean {
  return MISSING_VALUES.has(value.trim().toLowerCase());
}

function strictNumber(value: string): number | undefined {
  const trimmed = value.trim();
  if (!trimmed || !/^[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?$/u.test(trimmed)) return undefined;
  const parsed = Number(trimmed);
  return Number.isFinite(parsed) ? parsed : undefined;
}

function isDateValue(value: string): boolean {
  const trimmed = value.trim();
  if (!ISO_DATE.test(trimmed)) return false;
  return !Number.isNaN(Date.parse(trimmed));
}

function deterministicIndex(rowIndex: number, capacity: number): number {
  let value = (rowIndex + 1) >>> 0;
  value ^= value << 13;
  value ^= value >>> 17;
  value ^= value << 5;
  return (value >>> 0) % capacity;
}

function addQuantileSample(acc: ColumnAccumulator, value: number, observation: number): void {
  if (acc.quantileSample.length < QUANTILE_SAMPLE_LIMIT) {
    acc.quantileSample.push(value);
    return;
  }
  const candidate = deterministicIndex(observation, observation + 1);
  if (candidate < QUANTILE_SAMPLE_LIMIT) acc.quantileSample[candidate] = value;
}

function quantile(sorted: readonly number[], p: number): number | undefined {
  if (!sorted.length) return undefined;
  if (sorted.length === 1) return sorted[0];
  const position = (sorted.length - 1) * p;
  const low = Math.floor(position);
  const high = Math.ceil(position);
  if (low === high) return sorted[low];
  const fraction = position - low;
  return sorted[low]! * (1 - fraction) + sorted[high]! * fraction;
}

function uniqueHeaders(row: readonly string[]): string[] {
  const seen = new Map<string, number>();
  return row.map((raw, index) => {
    const base = raw.trim() || `column_${index + 1}`;
    const count = (seen.get(base) ?? 0) + 1;
    seen.set(base, count);
    return count === 1 ? base : `${base}_${count}`;
  });
}

function latexEscape(value: string): string {
  const map: Record<string, string> = {
    '\\': '\\textbackslash{}',
    '&': '\\&',
    '%': '\\%',
    '$': '\\$',
    '#': '\\#',
    '_': '\\_',
    '{': '\\{',
    '}': '\\}',
    '~': '\\textasciitilde{}',
    '^': '\\textasciicircum{}',
  };
  return [...value].map((char) => map[char] ?? char).join('');
}

function latexCell(value: string): string {
  const normalized = value.replace(/\s+/gu, ' ').trim();
  const bounded = normalized.length <= 160 ? normalized : `${normalized.slice(0, 157)}...`;
  return latexEscape(bounded);
}

function tableCaption(path: ProjectFilePath): string {
  const leaf = String(path).split('/').at(-1) ?? 'Results';
  const base = leaf.replace(/\.[^.]+$/u, '').replace(/[_-]+/gu, ' ').trim();
  return base ? `${base[0]?.toUpperCase() ?? ''}${base.slice(1)}` : 'Results';
}

function tableLabel(path: ProjectFilePath): string {
  const leaf = String(path).split('/').at(-1) ?? 'results';
  const slug = leaf.replace(/\.[^.]+$/u, '').toLowerCase().replace(/[^a-z0-9]+/gu, '-').replace(/^-+|-+$/gu, '');
  return `tab:${slug || 'results'}`;
}

function* parseDelimited(text: string, delimiter: ',' | '\t'): Generator<string[]> {
  let field = '';
  let row: string[] = [];
  let quoted = false;
  for (let index = 0; index < text.length; index += 1) {
    const ch = text[index]!;
    if (quoted) {
      if (ch === '"') {
        if (text[index + 1] === '"') {
          field += '"';
          index += 1;
        } else {
          quoted = false;
        }
      } else {
        field += ch;
      }
      continue;
    }
    if (ch === '"' && field.length === 0) {
      quoted = true;
    } else if (ch === delimiter) {
      row.push(field);
      field = '';
    } else if (ch === '\n') {
      row.push(field.endsWith('\r') ? field.slice(0, -1) : field);
      yield row;
      row = [];
      field = '';
    } else {
      field += ch;
    }
  }
  if (field.length || row.length) {
    row.push(field.endsWith('\r') ? field.slice(0, -1) : field);
    yield row;
  }
}

function createAccumulator(name: string, index: number): ColumnAccumulator {
  return {
    name,
    index,
    missingCount: 0,
    nonMissingCount: 0,
    couldBeInteger: true,
    couldBeNumber: true,
    couldBeBoolean: true,
    couldBeDate: true,
    numericCount: 0,
    numericMean: 0,
    numericM2: 0,
    numericMin: Number.POSITIVE_INFINITY,
    numericMax: Number.NEGATIVE_INFINITY,
    quantileSample: [],
    categories: new Map(),
    categoriesCapped: false,
  };
}

function observe(acc: ColumnAccumulator, raw: string): void {
  const trimmed = raw.trim();
  if (isMissing(trimmed)) {
    acc.missingCount += 1;
    return;
  }
  acc.nonMissingCount += 1;
  const numeric = strictNumber(trimmed);
  if (numeric === undefined) {
    acc.couldBeNumber = false;
    acc.couldBeInteger = false;
  } else {
    if (!Number.isInteger(numeric)) acc.couldBeInteger = false;
    acc.numericCount += 1;
    const delta = numeric - acc.numericMean;
    acc.numericMean += delta / acc.numericCount;
    acc.numericM2 += delta * (numeric - acc.numericMean);
    acc.numericMin = Math.min(acc.numericMin, numeric);
    acc.numericMax = Math.max(acc.numericMax, numeric);
    addQuantileSample(acc, numeric, acc.numericCount - 1);
  }
  if (!BOOLEAN_VALUES.has(trimmed.toLowerCase())) acc.couldBeBoolean = false;
  if (!isDateValue(trimmed)) acc.couldBeDate = false;

  const category = compactCell(trimmed);
  if (!acc.categoriesCapped || acc.categories.has(category)) {
    const current = acc.categories.get(category);
    if (current !== undefined) acc.categories.set(category, current + 1);
    else if (acc.categories.size < CATEGORY_LIMIT) acc.categories.set(category, 1);
    else acc.categoriesCapped = true;
  }
}

function finalizeColumn(acc: ColumnAccumulator, statistics: boolean): DatasetColumnSummary {
  let inferredType: DatasetColumnType = 'string';
  if (acc.nonMissingCount === 0) inferredType = 'empty';
  else if (acc.couldBeInteger && acc.numericCount === acc.nonMissingCount) inferredType = 'integer';
  else if (acc.couldBeNumber && acc.numericCount === acc.nonMissingCount) inferredType = 'number';
  else if (acc.couldBeBoolean) inferredType = 'boolean';
  else if (acc.couldBeDate) inferredType = 'date';

  const topValues = [...acc.categories.entries()]
    .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
    .slice(0, TOP_VALUE_LIMIT)
    .map(([value, count]) => ({ value, count }));

  let numeric: DatasetNumericStatistics | undefined;
  if (statistics && acc.numericCount > 0 && (inferredType === 'integer' || inferredType === 'number')) {
    const sorted = [...acc.quantileSample].sort((a, b) => a - b);
    numeric = {
      count: acc.numericCount,
      min: acc.numericMin,
      max: acc.numericMax,
      mean: acc.numericMean,
      std: acc.numericCount > 1 ? Math.sqrt(acc.numericM2 / (acc.numericCount - 1)) : 0,
      q1: quantile(sorted, 0.25),
      median: quantile(sorted, 0.5),
      q3: quantile(sorted, 0.75),
      quantilesApproximate: acc.numericCount > QUANTILE_SAMPLE_LIMIT,
    };
  }

  return {
    index: acc.index,
    name: acc.name,
    inferredType,
    missingCount: acc.missingCount,
    nonMissingCount: acc.nonMissingCount,
    ...(numeric ? { numeric } : {}),
    uniqueCount: acc.categories.size,
    uniqueCountCapped: acc.categoriesCapped,
    topValues,
  };
}

export function inspectDelimitedDatasetBytes(
  path: ProjectFilePath,
  bytes: Uint8Array,
  options: DatasetInspectionOptions = {},
): DatasetInspection {
  const lower = String(path).toLowerCase();
  const format: DatasetFormat = lower.endsWith('.tsv') ? 'tsv' : 'csv';
  const delimiter = format === 'tsv' ? '\t' : ',';
  const sampleLimit = Math.min(MAX_SAMPLE_ROWS, Math.max(0, options.sampleRows ?? DEFAULT_SAMPLE_ROWS));
  const statistics = options.statistics !== false;
  const text = new TextDecoder('utf-8', { fatal: false }).decode(bytes).replace(/^\uFEFF/u, '');
  const iterator = parseDelimited(text, delimiter);
  const headerRow = iterator.next();
  if (headerRow.done || !headerRow.value.length) {
    return {
      path,
      format,
      delimiter,
      bytes: bytes.byteLength,
      rowCount: 0,
      columnCount: 0,
      columns: [],
      sampleRows: [],
      malformedRowCount: 0,
      warnings: ['Dataset is empty.'],
    };
  }
  if (headerRow.value.length > MAX_COLUMNS) throw new Error(`Dataset has ${headerRow.value.length} columns; Learnitive inspection currently supports at most ${MAX_COLUMNS}.`);
  const headers = uniqueHeaders(headerRow.value);
  const accumulators = headers.map(createAccumulator);
  const samples: string[][] = [];
  let rowCount = 0;
  let malformedRowCount = 0;
  let truncatedCellPreview = false;
  const malformedExamples: number[] = [];
  for (const row of iterator) {
    if (row.length === 1 && row[0] === '') continue;
    rowCount += 1;
    if (row.length !== headers.length) {
      malformedRowCount += 1;
      if (malformedExamples.length < 5) malformedExamples.push(rowCount + 1);
    }
    const normalized = Array.from({ length: headers.length }, (_, index) => row[index] ?? '');
    if (normalized.some((value) => value.length > MAX_CELL_PREVIEW)) truncatedCellPreview = true;
    if (samples.length < sampleLimit) samples.push(normalized.map(compactCell));
    for (let index = 0; index < accumulators.length; index += 1) observe(accumulators[index]!, normalized[index] ?? '');
  }
  const warnings: string[] = [];
  if (malformedRowCount) warnings.push(`${malformedRowCount} row(s) have a different field count from the header${malformedExamples.length ? ` (examples: ${malformedExamples.join(', ')})` : ''}.`);
  if (accumulators.some((item) => item.categoriesCapped)) warnings.push(`Unique-value counting is capped at ${CATEGORY_LIMIT} values per column; high-cardinality counts are lower bounds.`);
  if (accumulators.some((item) => item.numericCount > QUANTILE_SAMPLE_LIMIT)) warnings.push(`Quantiles use a deterministic sample of at most ${QUANTILE_SAMPLE_LIMIT} numeric observations per column; mean/std/min/max remain full-dataset calculations.`);
  if (truncatedCellPreview) warnings.push(`Sample/top-value cell previews are capped at ${MAX_CELL_PREVIEW} characters; analysis runtimes still receive the canonical dataset file.`);
  return {
    path,
    format,
    delimiter,
    bytes: bytes.byteLength,
    rowCount,
    columnCount: headers.length,
    columns: accumulators.map((item) => finalizeColumn(item, statistics)),
    sampleRows: samples,
    malformedRowCount,
    warnings,
  };
}

export function datasetBytesToLatexTable(
  path: ProjectFilePath,
  bytes: Uint8Array,
  options: DatasetLatexTableOptions = {},
): DatasetLatexTable {
  const inspection = inspectDelimitedDatasetBytes(path, bytes, { sampleRows: 0, statistics: false });
  if (inspection.columnCount === 0) throw new Error('Cannot create a LaTeX table from an empty dataset.');
  const maxRows = Math.min(100, Math.max(1, options.maxRows ?? 30));
  const maxColumns = Math.min(24, Math.max(1, options.maxColumns ?? 12));
  const includedColumns = Math.min(inspection.columnCount, maxColumns);
  const text = new TextDecoder('utf-8', { fatal: false }).decode(bytes).replace(/^\uFEFF/u, '');
  const iterator = parseDelimited(text, inspection.delimiter);
  const header = iterator.next();
  if (header.done) throw new Error('Cannot create a LaTeX table from an empty dataset.');
  const headers = uniqueHeaders(header.value).slice(0, includedColumns);
  const rows: string[][] = [];
  for (const row of iterator) {
    if (row.length === 1 && row[0] === '') continue;
    if (rows.length >= maxRows) break;
    rows.push(Array.from({ length: includedColumns }, (_, index) => row[index] ?? ''));
  }
  const alignment = Array.from({ length: includedColumns }, () => 'l').join('');
  const caption = latexEscape(options.caption?.trim() || tableCaption(path));
  const label = latexEscape(options.label?.trim() || tableLabel(path));
  const lines = [
    '\\begin{table}[htbp]',
    '  \\centering',
    '  \\small',
    `  \\begin{tabular}{${alignment}}`,
    '    \\hline',
    `    ${headers.map((value) => `\\textbf{${latexCell(value)}}`).join(' & ')} \\\\`,
    '    \\hline',
    ...rows.map((row) => `    ${row.map(latexCell).join(' & ')} \\\\`),
    '    \\hline',
    '  \\end{tabular}',
    `  \\caption{${caption}}`,
    `  \\label{${label}}`,
    '\\end{table}',
  ];
  return {
    path,
    latex: lines.join('\n'),
    rowCount: inspection.rowCount,
    columnCount: inspection.columnCount,
    includedRows: rows.length,
    includedColumns,
    truncatedRows: inspection.rowCount > rows.length,
    truncatedColumns: inspection.columnCount > includedColumns,
  };
}

export class ProjectDatasetService extends BaseService {
  static deps = ['projectStorage'] as const;

  constructor(
    context: BaseServiceContext,
    private readonly dependencies: { projectStorage: ProjectStorageService },
  ) {
    super(SERVICE_NAME.projectDatasetService, context, dependencies);
  }

  hookMount(): void {}

  async inspect(
    wsName: string,
    pathInput: string,
    options: DatasetInspectionOptions = {},
    signal?: AbortSignal,
  ): Promise<DatasetInspection> {
    await this.mountPromise;
    signal?.throwIfAborted();
    const path = projectFilePath(pathInput.replace(/^\/workspace\//u, '').replace(/^\.\//u, ''));
    if (!/\.(?:csv|tsv)$/iu.test(path)) throw new Error('Dataset inspection currently supports CSV and TSV files.');
    const provider = await this.dependencies.projectStorage.getProvider(wsName);
    const entries = await provider.listEntries(signal);
    const entry = entries.find((item) => item.path === path);
    if (!entry || entry.kind !== 'file') throw new Error(`Dataset not found: ${path}`);
    const maxBytes = Math.max(1, options.maxBytes ?? DEFAULT_MAX_DATASET_BYTES);
    if (entry.size !== undefined && entry.size > maxBytes) {
      throw new Error(`Dataset is ${entry.size} bytes; deterministic inspection is currently limited to ${maxBytes} bytes because project storage does not yet expose streaming reads.`);
    }
    const bytes = await provider.readFile(path, signal);
    if (!bytes) throw new Error(`Dataset not found: ${path}`);
    if (bytes.byteLength > maxBytes) throw new Error(`Dataset is ${bytes.byteLength} bytes; deterministic inspection is currently limited to ${maxBytes} bytes.`);
    return inspectDelimitedDatasetBytes(path, bytes, options);
  }

  async toLatexTable(
    wsName: string,
    pathInput: string,
    options: DatasetLatexTableOptions = {},
    signal?: AbortSignal,
  ): Promise<DatasetLatexTable> {
    await this.mountPromise;
    signal?.throwIfAborted();
    const path = projectFilePath(pathInput.replace(/^\/workspace\//u, '').replace(/^\.\//u, ''));
    if (!/\.(?:csv|tsv)$/iu.test(path)) throw new Error('LaTeX table conversion currently supports CSV and TSV files.');
    const provider = await this.dependencies.projectStorage.getProvider(wsName);
    const entries = await provider.listEntries(signal);
    const entry = entries.find((item) => item.path === path);
    if (!entry || entry.kind !== 'file') throw new Error(`Dataset not found: ${path}`);
    const maxBytes = Math.max(1, options.maxBytes ?? DEFAULT_MAX_DATASET_BYTES);
    if (entry.size !== undefined && entry.size > maxBytes) throw new Error(`Dataset is ${entry.size} bytes; LaTeX table conversion is currently limited to ${maxBytes} bytes.`);
    const bytes = await provider.readFile(path, signal);
    if (!bytes) throw new Error(`Dataset not found: ${path}`);
    if (bytes.byteLength > maxBytes) throw new Error(`Dataset is ${bytes.byteLength} bytes; LaTeX table conversion is currently limited to ${maxBytes} bytes.`);
    return datasetBytesToLatexTable(path, bytes, options);
  }
}