Turn lsh into a standalone crate (#958)

* Remove `stdext` from `lsh`
  This required a huge change from raw pointers for graphs to vecs and
  indices. To make it less ugly (due to borrowing), I refactored the
  data common between frontend and backend into `ir.rs`.
* Move `lsh-bin` into `lsh` as its main binary

The assembly output remains unmodified (as is the overall approach).
The Rust output lost its mermaid diagrams, because they were in the
way and outlived their usefulness anyway (they are too large).

Author: lhecker

.github/workflows/ci.yml

@@ -44,8 +44,8 @@ jobs:
       - name: Install Rust
         run: rustup toolchain install nightly --no-self-update --profile minimal --component rust-src,rustfmt,clippy
       - name: Check formatting
-        run: cargo fmt --all -- --check
+        run: cargo +nightly fmt --all -- --check
       - name: Run tests
-        run: cargo test --all-features --all-targets
+        run: cargo +nightly test --all-features --all-targets
       - name: Run clippy
-        run: cargo clippy --workspace --all-features --all-targets -- --no-deps --deny warnings
+        run: cargo +nightly clippy --workspace --all-features --all-targets -- --no-deps --deny warnings

Cargo.lock

@@ -25,9 +25,3 @@ incremental = true          # Improves re-compile times
 [profile.bench]
 codegen-units = 16          # Make compiling criterion faster (16 is the default, but profile.release sets it to 1)
 lto = "thin"                # Similarly, speed up linking by a ton
-
-[workspace.dependencies]
-edit = { path = "./crates/edit" }
-lsh = { path = "./crates/lsh" }
-stdext = { path = "./crates/stdext" }
-unicode-gen = { path = "./crates/unicode-gen" }

Cargo.toml

@@ -5,6 +5,5 @@ This directory contains the crates that make up Edit and its supporting tooling.
 * `edit`: Main editor binary and library<br>
   It is split apart into a library to allow for benchmarks.
 * `lsh`: Syntax-highlighting compiler and runtime
-* `lsh-bin`: A small CLI for experimenting with and debugging LSH output
 * `stdext`: Shared utility code used across the workspace
 * `unicode-gen`: Code generation utilities for Unicode LUTs

crates/README.md

@@ -1,5 +1,6 @@
 [package]
 name = "edit"
+description = "A simple editor for simple needs."
 version = "2.0.0"
 
 edition.workspace = true
@@ -19,15 +20,14 @@ harness = false
 debug-latency = []
 
 [dependencies]
-lsh.workspace = true
-stdext.workspace = true
+lsh = { path = "../lsh", default-features = false, features = ["compiler", "glob"] }
+stdext = { path = "../stdext" }
 
 [target.'cfg(unix)'.dependencies]
 libc = "0.2"
 
 [build-dependencies]
-stdext.workspace = true
-lsh.workspace = true
+lsh = { path = "../lsh", default-features = false, features = ["compiler", "glob"] }
 # The default toml crate bundles its dependencies with bad compile times. Thanks.
 # Thankfully toml-span exists. FWIW the alternative is yaml-rust (without the 2 suffix).
 toml-span = { version = "0.7", default-features = false }

crates/edit/Cargo.toml

@@ -6,13 +6,13 @@ use std::io::Cursor;
 use std::path::Path;
 use std::{mem, vec};
 
+use ::lsh::glob;
 use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
 use edit::helpers::*;
 use edit::{buffer, hash, json, lsh, oklab, simd, unicode};
 use stdext::arena::{self, scratch_arena};
 use stdext::collections::BVec;
 use stdext::float::parse_f64_approx;
-use stdext::glob;
 use stdext::unicode::Utf8Chars;
 
 struct EditingTracePatch<'a>(usize, usize, &'a str);

crates/edit/benches/lib.rs

@@ -3,8 +3,6 @@
 
 #![allow(irrefutable_let_patterns)]
 
-use stdext::arena::scratch_arena;
-
 use crate::helpers::env_opt;
 
 mod helpers;
@@ -18,8 +16,6 @@ enum TargetOs {
 }
 
 fn main() {
-    stdext::arena::init(128 * 1024 * 1024).unwrap();
-
     let target_os = match env_opt("CARGO_CFG_TARGET_OS").as_str() {
         "windows" => TargetOs::Windows,
         "macos" | "ios" => TargetOs::MacOS,
@@ -34,13 +30,11 @@ fn main() {
 }
 
 fn compile_lsh() {
-    let scratch = scratch_arena(None);
-
     let lsh_path = lsh::compiler::builtin_definitions_path();
     let out_dir = env_opt("OUT_DIR");
     let out_path = format!("{out_dir}/lsh_definitions.rs");
 
-    let mut generator = lsh::compiler::Generator::new(&scratch);
+    let mut generator = lsh::compiler::Generator::new();
     match generator.read_directory(lsh_path).and_then(|_| generator.generate_rust()) {
         Ok(c) => std::fs::write(out_path, c).unwrap(),
         Err(err) => {

crates/edit/build/main.rs

@@ -12,6 +12,17 @@ use crate::{simd, unicode};
 
 const MAX_LINE_LEN: usize = 32 * KIBI;
 
+struct VecWithAlloc<'a, 'b, T> {
+    arena: &'a Arena,
+    vec: &'b mut BVec<'a, T>,
+}
+
+impl<'a, 'b, T> ExtendOne<T> for VecWithAlloc<'a, 'b, T> {
+    fn extend_one(&mut self, value: T) {
+        self.vec.push(self.arena, value);
+    }
+}
+
 #[derive(Clone)]
 pub struct Highlighter<'a> {
     doc: &'a dyn ReadableDocument,
@@ -71,7 +82,8 @@ impl<'doc> Highlighter<'doc> {
         }
 
         let line = unicode::strip_newline(line);
-        let mut res = self.runtime.parse_next_line(arena, line);
+        let mut res: BVec<'a, Highlight<HighlightKind>> = BVec::empty();
+        self.runtime.parse_next_line(line, VecWithAlloc { arena, vec: &mut res });
 
         // Adjust the range to account for the line offset.
         for h in res.iter_mut() {

crates/edit/src/lsh/highlighter.rs

@@ -11,8 +11,8 @@ use std::path::Path;
 
 pub use definitions::{FILE_ASSOCIATIONS, HighlightKind, LANGUAGES};
 pub use highlighter::*;
+use lsh::glob::glob_match;
 pub use lsh::runtime::Language;
-use stdext::glob::glob_match;
 
 pub fn process_file_associations<T>(
     associations: &[(T, &'static Language)],