Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,6 @@ skills = [
teach = [
# Pinned to a commit: uv.lock is gitignored, so tracking colloquium main
# let an upstream change break unrelated deploys. This is colloquium main
# after the footnote-citation fix (natolambert/colloquium#70); bump deliberately.
"colloquium @ git+https://github.com/natolambert/colloquium.git@c8d4b1c9e41b3f6a14df074b91ae3568be2c4e94",
# with the show_animations flag (natolambert/colloquium#76); bump deliberately.
"colloquium @ git+https://github.com/natolambert/colloquium.git@440b85e43348bbe3a422745e2294139d3658c449",
]
153 changes: 153 additions & 0 deletions teach/Cornell-CS4789-2026/README.md

Large diffs are not rendered by default.

Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added teach/Cornell-CS4789-2026/assets/rl.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
159 changes: 159 additions & 0 deletions teach/Cornell-CS4789-2026/refs.bib
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
@article{ionides2008truncated,
title={Truncated importance sampling},
author={Ionides, Edward L},
journal={Journal of Computational and Graphical Statistics},
volume={17},
number={2},
pages={295--311},
year={2008}
}

@misc{marek2026score,
title = {Score Centering Stabilizes Off-policy Reinforcement Learning},
author = {Marek, Martin and Ryabinin, Max},
year = {2026},
eprint = {2609.20807},
archivePrefix = {arXiv},
primaryClass = {cs.LG},
doi = {10.48550/arXiv.2609.20807},
url = {https://arxiv.org/abs/2609.20807}
}

@misc{minimax2025minimax_m1,
title={MiniMax-M1: Scaling Test-Time Compute Efficiently with Lightning Attention},
author={MiniMax},
year={2025},
eprint={2506.13585},
archivePrefix={arXiv},
primaryClass={cs.CL},
doi={10.48550/arXiv.2506.13585},
url={https://arxiv.org/abs/2506.13585},
}

@inproceedings{noukhovitch2024asynchronous,
title={Asynchronous RLHF: Faster and More Efficient Off-Policy RL for Language Models},
booktitle = {International Conference on Learning Representations (ICLR)},
author={Noukhovitch, Michael and Huang, Shengyi and Xhonneux, Sophie and Hosseini, Arian and Agarwal, Rishabh and Courville, Aaron},
year = {2025}
}

@article{shao2024deepseekmath,
title={Deepseekmath: Pushing the limits of mathematical reasoning in open language models},
author={Shao, Zhihong and Wang, Peiyi and Zhu, Qihao and Xu, Runxin and Song, Junxiao and Bi, Xiao and Zhang, Haowei and Zhang, Mingchuan and Li, YK and Wu, Y and others},
journal={arXiv preprint arXiv:2402.03300},
year={2024}
}

@article{williams1992simple,
title={Simple statistical gradient-following algorithms for connectionist reinforcement learning},
author={Williams, Ronald J},
journal={Machine learning},
volume={8},
pages={229--256},
year={1992},
publisher={Springer}
}

@misc{yao2025offpolicy,
title={Your Efficient RL Framework Secretly Brings You Off-Policy RL Training},
author={Yao, Feng and Liu, Liyuan and Zhang, Dinghuai and Dong, Chengyu and Shang, Jingbo and Gao, Jianfeng},
url={https://fengyao.notion.site/off-policy-rl},
year={2025}
}

@misc{tinker2026stablereinforce,
title = {{StableReinforce}},
author = {{Thinking Machines Lab}},
howpublished = {Tinker documentation},
url = {https://tinker-docs.thinkingmachines.ai/tinker/losses/stable-reinforce/},
year = {2026}
}

@misc{marek2026stablereinforcethread,
title = {Thread reverse-engineering {StableReinforce} from its gradients},
author = {Marek, Martin},
howpublished = {Thread on X},
url = {https://x.com/mrtnm/status/2114145574987702525},
year = {2026}
}

@misc{ricursive2026ripple,
title = {Introducing {Ripple-65B} Preview, a Frontier Open-Weight Model},
author = {{Ricursive Intelligence}},
url = {https://www.ricursive.com/ripple},
year = {2026}
}

@misc{semianalysis2026computeshare,
title = {Share of Frontier Lab Compute by Workload},
author = {{SemiAnalysis}},
howpublished = {Chart reported by OfficeChai},
url = {https://officechai.com/ai/while-share-of-inference-has-held-steady-over-the-last-2-years-pre-training-compute-has-moved-to-post-training-semi-analysis/},
year = {2026}
}

@article{guo2025deepseeknature,
title = {{DeepSeek-R1} incentivizes reasoning in {LLMs} through reinforcement learning},
author = {Guo, Daya and Yang, Dejian and Zhang, Haowei and others},
journal = {Nature},
volume = {645},
pages = {633--638},
doi = {10.1038/s41586-025-09422-z},
year = {2025}
}

@misc{deepseekai2025deepseekv3technicalreport,
title = {{DeepSeek-V3} Technical Report},
author = {{DeepSeek-AI}},
url = {https://arxiv.org/abs/2412.19437},
year = {2025}
}

@misc{reflection2026beam,
title = {Introducing {Beam}},
author = {{Reflection AI}},
url = {https://reflection.ai/blog/introducing-beam},
year = {2026}
}

@misc{tml2026inkling2small,
title = {Introducing {Inkling 2 Small}},
author = {{Thinking Machines Lab}},
url = {https://thinkingmachines.ai/news/inkling-2-small},
year = {2026}
}

@inproceedings{Vaswani2017AttentionIA,
title = {Attention Is All You Need},
author = {Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N. and Kaiser, {\L}ukasz and Polosukhin, Illia},
booktitle = {Advances in Neural Information Processing Systems 30 (NeurIPS 2017)},
year = {2017},
url = {https://papers.nips.cc/paper/7181-attention-is-all-you-need}
}

@article{ivison2026tmax,
title = {{TMax}: A Simple Recipe for Terminal Agents},
author = {Ivison, Hamish and Yin, Junjie Oscar and Shao, Rulin and Xiao, Teng and Lambert, Nathan and Hajishirzi, Hannaneh},
journal = {arXiv preprint arXiv:2606.23321},
year = {2026},
eprint = {2606.23321},
url = {https://arxiv.org/abs/2606.23321}
}

@misc{teamolmo2025olmo3,
title = {Olmo 3},
author = {Team Olmo and Allyson Ettinger and Amanda Bertsch and Bailey Kuehl and David Graham and David Heineman and Dirk Groeneveld and Faeze Brahman and Finbarr Timbers and Hamish Ivison and Jacob Morrison and Jake Poznanski and Kyle Lo and Luca Soldaini and Matt Jordan and Mayee Chen and Michael Noukhovitch and Nathan Lambert and Pete Walsh and Pradeep Dasigi and Robert Berry and Saumya Malik and Saurabh Shah and Scott Geng and Shane Arora and Shashank Gupta and Taira Anderson and Teng Xiao and Tyler Murray and Tyler Romero and Victoria Graf and Akari Asai and Akshita Bhagia and Alexander Wettig and Alisa Liu and Aman Rangapur and Chloe Anastasiades and Costa Huang and Dustin Schwenk and Harsh Trivedi and Ian Magnusson and Jaron Lochner and Jiacheng Liu and Lester James V. Miranda and Maarten Sap and Malia Morgan and Michael Schmitz and Michal Guerquin and Michael Wilson and Regan Huff and Ronan Le Bras and Rui Xin and Rulin Shao and Sam Skjonsberg and Shannon Zejiang Shen and Shuyue Stella Li and Tucker Wilde and Valentina Pyatkin and Will Merrill and Yapei Chang and Yuling Gu and Zhiyuan Zeng and Ashish Sabharwal and Luke Zettlemoyer and Pang Wei Koh and Ali Farhadi and Noah A. Smith and Hannaneh Hajishirzi},
year = {2025},
eprint = {2512.13961},
archiveprefix = {arXiv},
primaryclass = {cs.CL},
url = {https://arxiv.org/abs/2512.13961}
}

@article{piche2025pipelinerl,
title = {{PipelineRL}: Faster On-policy Reinforcement Learning for Long Sequence Generation},
author = {Piché, Alexandre and Kamalloo, Ehsan and Pardinas, Rafael and Chen, Xiaoyin and Bahdanau, Dzmitry},
journal = {arXiv preprint arXiv:2509.19128},
year = {2025},
url = {https://arxiv.org/abs/2509.19128}
}
Loading
Loading