introduce rustpython-compiler-source

qingshi163 · Sep 10, 2024 · Oct 7, 2024 · Oct 7, 2024 · Oct 15, 2024 · Oct 15, 2024
commit 38575bd55d82871b4542c14a0261ba8cff57cbbd
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -109,6 +109,7 @@ repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
 
 [workspace.dependencies]
+rustpython-compiler-source = { path = "compiler/source" }
 rustpython-compiler-core = { path = "compiler/core", version = "0.4.0" }
 rustpython-compiler = { path = "compiler", version = "0.4.0" }
 rustpython-codegen = { path = "compiler/codegen", version = "0.4.0" }

diff --git a/compiler/Cargo.toml b/compiler/Cargo.toml
@@ -11,10 +11,12 @@ license.workspace = true
 [dependencies]
 rustpython-codegen = { workspace = true }
 rustpython-compiler-core = { workspace = true }
+rustpython-compiler-source = { workspace = true }
 # rustpython-parser = { workspace = true }
 ruff_python_parser = { workspace = true }
 ruff_python_ast = { workspace = true }
 ruff_source_file = { workspace = true }
+ruff_text_size = { workspace = true }
 thiserror = { workspace = true }
 
 [lints]

diff --git a/compiler/codegen/Cargo.toml b/compiler/codegen/Cargo.toml
@@ -13,6 +13,7 @@ license.workspace = true
 # rustpython-ast = { workspace = true, features=["unparse", "constant-optimization"] }
 # rustpython-parser-core = { workspace = true }
 rustpython-compiler-core = { workspace = true }
+rustpython-compiler-source = {workspace = true }
 ruff_python_parser = { workspace = true }
 ruff_python_ast = { workspace = true }
 ruff_text_size = { workspace = true }

diff --git a/compiler/codegen/src/compile.rs b/compiler/codegen/src/compile.rs
@@ -23,7 +23,7 @@ use ruff_python_ast::{
     Parameters, Stmt, StmtImport, StmtImportFrom, TypeParam, TypeParamTypeVar, TypeParams, UnaryOp,
     WithItem,
 };
-use ruff_source_file::{LineIndex, OneIndexed, SourceCode};
+use ruff_source_file::{LineIndex, OneIndexed};
 use ruff_text_size::{Ranged, TextRange};
 // use rustpython_ast::located::{self as located_ast, Located};
 use rustpython_compiler_core::{
@@ -33,6 +33,7 @@ use rustpython_compiler_core::{
     },
     Mode,
 };
+use rustpython_compiler_source::SourceCode;
 // use rustpython_parser_core::source_code::{LineNumber, SourceLocation};
 use std::borrow::Cow;
 
@@ -62,9 +63,7 @@ fn is_forbidden_name(name: &str) -> bool {
 struct Compiler<'src> {
     code_stack: Vec<ir::CodeInfo>,
     symbol_table_stack: Vec<SymbolTable>,
-    source_path: String,
-    source_text: &'src str,
-    source_index: LineIndex,
+    source_code: SourceCode<'src>,
     // current_source_location: SourceLocation,
     current_source_range: TextRange,
     qualified_path: Vec<String>,
@@ -113,18 +112,13 @@ enum ComprehensionType {
 /// Compile an Mod produced from rustpython_parser::parse()
 pub fn compile_top(
     ast: ruff_python_ast::Mod,
-    source_path: String,
-    source_text: &str,
+    source_code: SourceCode,
     mode: Mode,
     opts: CompileOpts,
 ) -> CompileResult<CodeObject> {
     match ast {
-        ruff_python_ast::Mod::Module(module) => {
-            compile_program(&module, source_path, source_text, opts)
-        }
-        ruff_python_ast::Mod::Expression(expr) => {
-            compile_expression(&expr, source_path, source_text, opts)
-        }
+        ruff_python_ast::Mod::Module(module) => compile_program(&module, source_code, opts),
+        ruff_python_ast::Mod::Expression(expr) => compile_expression(&expr, source_code, opts),
     }
     // match ast {
     //     Mod::Module(ModModule { body, .. }) => {
@@ -169,21 +163,12 @@ pub fn compile_top(
 /// Compile a standard Python program to bytecode
 pub fn compile_program(
     ast: &ModModule,
-    source_path: String,
-    source_text: &str,
+    source_code: SourceCode,
     opts: CompileOpts,
 ) -> CompileResult<CodeObject> {
-    let source_index = LineIndex::from_source_text(source_text);
-    let source_code = SourceCode::new(source_text, &source_index);
-    let symbol_table = SymbolTable::scan_program(ast, source_code)
-        .map_err(|e| e.into_codegen_error(source_path.clone()))?;
-    let mut compiler = Compiler::new(
-        opts,
-        source_path,
-        source_text,
-        source_index,
-        "<module>".to_owned(),
-    );
+    let symbol_table = SymbolTable::scan_program(ast, source_code.clone())
+        .map_err(|e| e.into_codegen_error(source_code.path.to_owned()))?;
+    let mut compiler = Compiler::new(opts, source_code, "<module>".to_owned());
     compiler.compile_program(ast, symbol_table)?;
     let code = compiler.pop_code_object();
     Ok(code)
@@ -230,21 +215,12 @@ pub fn compile_program(
 
 pub fn compile_expression(
     ast: &ModExpression,
-    source_path: String,
-    source_text: &str,
+    source_code: SourceCode,
     opts: CompileOpts,
 ) -> CompileResult<CodeObject> {
-    let source_index = LineIndex::from_source_text(source_text);
-    let source_code = SourceCode::new(source_text, &source_index);
-    let symbol_table = SymbolTable::scan_expr(ast, source_code)
-        .map_err(|e| e.into_codegen_error(source_path.clone()))?;
-    let mut compiler = Compiler::new(
-        opts,
-        source_path,
-        source_text,
-        source_index,
-        "<module>".to_owned(),
-    );
+    let symbol_table = SymbolTable::scan_expr(ast, source_code.clone())
+        .map_err(|e| e.into_codegen_error(source_code.path.to_owned()))?;
+    let mut compiler = Compiler::new(opts, source_code, "<module>".to_owned());
     compiler.compile_eval(ast, symbol_table)?;
     let code = compiler.pop_code_object();
     Ok(code)
@@ -266,19 +242,13 @@ macro_rules! emit {
 }
 
 impl<'src> Compiler<'src> {
-    fn new(
-        opts: CompileOpts,
-        source_path: String,
-        source_text: &'src str,
-        source_index: LineIndex,
-        code_name: String,
-    ) -> Self {
+    fn new(opts: CompileOpts, source_code: SourceCode<'src>, code_name: String) -> Self {
         let module_code = ir::CodeInfo {
             flags: bytecode::CodeFlags::NEW_LOCALS,
             posonlyarg_count: 0,
             arg_count: 0,
             kwonlyarg_count: 0,
-            source_path: source_path.clone(),
+            source_path: source_code.path.to_owned(),
             first_line_number: OneIndexed::MIN,
             obj_name: code_name,
 
@@ -293,9 +263,7 @@ impl<'src> Compiler<'src> {
         Compiler {
             code_stack: vec![module_code],
             symbol_table_stack: Vec::new(),
-            source_path,
-            source_text,
-            source_index,
+            source_code,
             // current_source_location: SourceLocation::default(),
             current_source_range: TextRange::default(),
             qualified_path: Vec::new(),
@@ -317,11 +285,11 @@ impl Compiler<'_> {
         self.error_ranged(error, self.current_source_range)
     }
     fn error_ranged(&mut self, error: CodegenErrorType, range: TextRange) -> CodegenError {
-        let location = self.source_index.source_location(range.start(), self.source_text);
+        let location = self.source_code.source_location(range.start());
         CodegenError {
             error,
             location: Some(location),
-            source_path: self.source_path.clone(),
+            source_path: self.source_code.path.to_owned(),
         }
     }
 
@@ -353,7 +321,7 @@ impl Compiler<'_> {
         kwonlyarg_count: u32,
         obj_name: String,
     ) {
-        let source_path = self.source_path.clone();
+        let source_path = self.source_code.path.to_owned();
         let first_line_number = self.get_source_line_number();
 
         let table = self.push_symbol_table();
@@ -1470,7 +1438,7 @@ impl Compiler<'_> {
             let symbol = table.lookup(var).unwrap_or_else(|| {
                 panic!(
                     "couldn't look up var {} in {} in {}",
-                    var, code.obj_name, self.source_path
+                    var, code.obj_name, self.source_code.path
                 )
             });
             let parent_code = self.code_stack.last().unwrap();
@@ -3018,7 +2986,7 @@ impl Compiler<'_> {
     // Low level helper functions:
     fn _emit(&mut self, instr: Instruction, arg: OpArg, target: ir::BlockIdx) {
         let range = self.current_source_range;
-        let location = self.source_index.source_location(range.start(), self.source_text);
+        let location = self.source_code.source_location(range.start());
         // TODO: insert source filename
         self.current_block().instructions.push(ir::InstructionInfo {
             instr,
@@ -3108,7 +3076,7 @@ impl Compiler<'_> {
     }
 
     fn get_source_line_number(&mut self) -> OneIndexed {
-        self.source_index
+        self.source_code
             .line_index(self.current_source_range.start())
     }
 
@@ -3410,4 +3378,4 @@ for stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):
         ));
     }
 }
-*/
+*/
diff --git a/compiler/codegen/src/error.rs b/compiler/codegen/src/error.rs
@@ -1,8 +1,5 @@
 use std::fmt;
-
-use ruff_python_ast::Expr;
 use ruff_source_file::SourceLocation;
-use ruff_text_size::TextRange;
 use thiserror::Error;
 
 // pub type CodegenError = rustpython_parser_core::source_code::LocatedError<CodegenErrorType>;
@@ -15,7 +12,7 @@ pub struct CodegenError {
     pub source_path: String,
 }
 
-impl std::fmt::Display for CodegenError {
+impl fmt::Display for CodegenError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         // TODO:
         self.error.fmt(f)

diff --git a/compiler/codegen/src/symboltable.rs b/compiler/codegen/src/symboltable.rs
@@ -16,8 +16,8 @@ use ruff_python_ast::{
     self as ast, Comprehension, Decorator, Expr, ModExpression, ModModule, Parameter,
     ParameterWithDefault, Parameters, Stmt, TypeParam, TypeParamTypeVar, TypeParams,
 };
-use ruff_source_file::{SourceCode, SourceLocation};
 use ruff_text_size::{Ranged, TextRange};
+use rustpython_compiler_source::{SourceCode, SourceLocation};
 // use rustpython_ast::{self as ast, located::Located};
 // use rustpython_parser_core::source_code::{LineNumber, SourceLocation};
 use std::{borrow::Cow, fmt, thread::LocalKey};
@@ -551,12 +551,12 @@ enum SymbolUsage {
     Iter,
 }
 
-struct SymbolTableBuilder<'src, 'index> {
+struct SymbolTableBuilder<'src> {
     class_name: Option<String>,
     // Scope stack.
     tables: Vec<SymbolTable>,
     future_annotations: bool,
-    source_code: SourceCode<'src, 'index>,
+    source_code: SourceCode<'src>,
 }
 
 /// Enum to indicate in what mode an expression
@@ -572,8 +572,8 @@ enum ExpressionContext {
     IterDefinitionExp,
 }
 
-impl<'src, 'index> SymbolTableBuilder<'src, 'index> {
-    fn new(source_code: SourceCode<'src, 'index>) -> Self {
+impl<'src> SymbolTableBuilder<'src> {
+    fn new(source_code: SourceCode<'src>) -> Self {
         let mut this = Self {
             class_name: None,
             tables: vec![],
@@ -585,7 +585,7 @@ impl<'src, 'index> SymbolTableBuilder<'src, 'index> {
     }
 }
 
-impl SymbolTableBuilder<'_, '_> {
+impl SymbolTableBuilder<'_> {
     fn finish(mut self) -> Result<SymbolTable, SymbolTableError> {
         assert_eq!(self.tables.len(), 1);
         let mut symbol_table = self.tables.pop().unwrap();

diff --git a/compiler/source/Cargo.toml b/compiler/source/Cargo.toml
@@ -0,0 +1,16 @@
+[package]
+name = "rustpython-compiler-source"
+description = "RustPython Source and Index"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+repository.workspace = true
+license.workspace = true
+
+[dependencies]
+ruff_source_file = { workspace = true }
+ruff_text_size = { workspace = true }
+
+[lints]
+workspace = true
diff --git a/compiler/source/src/lib.rs b/compiler/source/src/lib.rs
@@ -0,0 +1,24 @@
+pub use ruff_source_file::{LineIndex, OneIndexed as LineNumber, SourceLocation};
+pub use ruff_text_size::TextSize;
+
+#[derive(Clone)]
+pub struct SourceCode<'src> {
+    pub path: &'src str,
+    pub text: &'src str,
+    pub index: LineIndex,
+}
+
+impl<'src> SourceCode<'src> {
+    pub fn new(path: &'src str, text: &'src str) -> Self {
+        let index = LineIndex::from_source_text(text);
+        Self { path, text, index }
+    }
+
+    pub fn line_index(&self, offset: TextSize) -> LineNumber {
+        self.index.line_index(offset)
+    }
+
+    pub fn source_location(&self, offset: TextSize) -> SourceLocation {
+        self.index.source_location(offset, self.text)
+    }
+}