base_checkpoints/d24_decoderstack/meta_005568.json

{
  "step": 5568,
  "val_bpb": 0.719042,
  "model_config": {
    "sequence_len": 2048,
    "vocab_size": 32768,
    "n_layer": 24,
    "n_head": 12,
    "n_kv_head": 12,
    "n_embd": 1536,
    "window_pattern": "SSSL"
  },
  "device_batch_size": 32,
  "max_seq_len": 2048,
  "total_batch_size": 1048576,
  "note": "DecoderStack-medium (d24) capture converted for nanochat. This meta was built after the fact from the run log (full_d24_w8.log); DecoderStack's capture writes only {step, code, weights} and does not emit a meta.json of its own. model_config is the nanochat GPTConfig equivalent of StackConfig -- the two architectures are the same model (both count 1,384,122,122 parameters). Sufficient for load/eval; NOT sufficient for resume: there is no dataloader_state_dict, loop_state, or optimizer state here, and DecoderStack's pre-tokenized binary loader has no state nanochat's tokenizing loader could consume. device_batch_size/max_seq_len are the nanochat-shaped restatement of micro_batch_tokens=65,536 (= 32 x 2048) -- DecoderStack trains varlen, so its micro-batch is a token count, not a sequence count. TOKENIZER: these weights use the 32k vocab shipped in the ChrisMcCormick/climbmix_32k_8_170 dataset repo (mirrored under tokenizer/ in this repo). It is NOT the same vocabulary as ChrisMcCormick/nanochat-varlen-d24-2026-03-22 -- 31,474 of 32,759 ids differ. Pairing these weights with that tokenizer produces garbage.",
  "stack_config": {
    "n_layers": 24,
    "d_model": 1536,
    "d_vocab": 32768,
    "d_smr_gate": 24,
    "n_q_heads": 12,
    "n_kv_heads": 12,
    "n_o_heads": 12,
    "d_qk": 128,
    "d_vo": 128,
    "seq_len": 2048,
    "short_win_size": 768,
    "full_ctxt_layers": [
      3,
      7,
      11,
      15,
      19,
      23
    ],
    "d_ve_gate": 12,
    "ve_layers": [
      1,
      3,
      5,
      7,
      9,
      11,
      13,
      15,
      17,
      19,
      21,
      23
    ],
    "d_mlp": 6144,
    "micro_batch_tokens": 65536,
    "total_batch_size": 1048576,
    "num_iterations": 5568,
    "max_num_docs": 192,
    "num_train_shards": 80,
    "val_tokens": 10485760,
    "val_loss_every": 250,
    "save_steps": [
      1950,
      5568
    ],
    "world_size": 8,
    "grad_accum_steps": 2,
    "precision": "bf16",
    "run_id": "2026-08-01_023219-d24"
  },
  "schedules": {
    "batch_lr_scale": 1.4142135623730951,
    "muon_weight_decay_peak": 0.059738,
    "warmup_steps": 40,
    "warmdown_ratio": 0.65,
    "final_lr_frac": 0.05,
    "matrix_lr": 0.02,
    "muon_momentum": "0.85->0.97 over 400 steps, cooled to 0.90",
    "muon_beta2": 0.9,
    "muon_ns_steps": 5,
    "unembedding_lr": 0.008,
    "embedding_lr": 0.3,
    "value_embedding_lr_mult": 0.5,
    "scalar_lr": 0.5,
    "smear_lr": 0.2,
    "adamw_eps": 1e-10
  },
  "results": {
    "core_metric": 0.2517,
    "min_val_bpb": 0.719042,
    "peak_memory_mib": 71569,
    "total_training_time_min": 110.74,
    "gpu": "8x NVIDIA H100 80GB HBM3"
  }
}