Add detailed logging to Python soname patcher

- Enhanced logging in fix-python-soname.js to track WASM execution
- Added verbose logging to Rust soname patcher for debugging
- Added Python library discovery diagnostics in CI pipeline
- Track file operations and show backup creation status

This will help diagnose why the soname patcher might not be working
correctly in the CI environment.

Author: Qard

.github/workflows/CI.yml

@@ -350,7 +350,22 @@ jobs:
             # Fix Python soname if WASI module is available
             if [ -f "fix-python-soname.wasm" ] && [ -f "fix-python-soname.js" ]; then
               echo "Fixing Python soname in .node file..."
+              # Debug: Check Python version and libraries available
+              echo "Checking Python installation..."
+              which python3 || true
+              python3 --version || true
+              echo "Looking for Python libraries..."
+              find /usr/lib -name "libpython*.so*" 2>/dev/null | head -20 || true
+              # Run the soname fixer
               node fix-python-soname.js || echo "Warning: Failed to fix Python soname"
+              # Check if backup was created
+              if [ -f "*.node.bak" ]; then
+                echo "Backup file created, soname fix was attempted"
+                ls -la *.node.bak || true
+              fi
+            else
+              echo "Soname fixer not found, skipping..."
+              ls -la fix-python-soname.* || true
             fi
             
             pnpm test

STREAM_AND_WS.md

@@ -0,0 +1,103 @@
+# STREAM_AND_WS.md
+
+## Future Design: Streaming and WebSocket Support
+
+This document captures design ideas for adding streaming HTTP responses and WebSocket support after the basic HTTP implementation is complete.
+
+### Streaming Response Support
+
+For streaming responses, we can add a separate method that returns a stream:
+
+```rust
+#[napi]
+impl PythonHandler {
+    #[napi]
+    pub async fn handle_request_stream(&self, request: Request) -> Result<ResponseStream> {
+        let (tx_sender, tx_receiver) = self.event_loop
+            .start_streaming_request(request)
+            .await?;
+        
+        Ok(ResponseStream { receiver: tx_receiver })
+    }
+}
+
+#[napi]
+pub struct ResponseStream {
+    receiver: mpsc::Receiver<Bytes>,
+}
+
+#[napi]
+impl ResponseStream {
+    #[napi]
+    pub async fn next_chunk(&mut self) -> Result<Option<Buffer>> {
+        match self.receiver.recv().await {
+            Some(bytes) => Ok(Some(bytes.into())),
+            None => Ok(None),
+        }
+    }
+}
+```
+
+### WebSocket with Async
+
+```rust
+#[napi]
+impl PythonHandler {
+    #[napi]
+    pub async fn handle_websocket(&self, request: WebSocketRequest) -> Result<WebSocketConnection> {
+        let connection = self.event_loop.create_websocket(request).await?;
+        Ok(connection)
+    }
+}
+
+#[napi]
+pub struct WebSocketConnection {
+    id: u64,
+    tx: mpsc::Sender<WebSocketMessage>,
+    rx: mpsc::Receiver<WebSocketMessage>,
+}
+
+#[napi]
+impl WebSocketConnection {
+    #[napi]
+    pub async fn send(&self, message: String) -> Result<()> {
+        self.tx.send(WebSocketMessage::Text(message)).await
+            .map_err(|_| Error::from_reason("Failed to send"))?;
+        Ok(())
+    }
+    
+    #[napi]
+    pub async fn receive(&mut self) -> Result<Option<String>> {
+        match self.rx.recv().await {
+            Some(WebSocketMessage::Text(text)) => Ok(Some(text)),
+            Some(WebSocketMessage::Binary(_)) => Ok(Some("[binary data]".to_string())),
+            None => Ok(None),
+        }
+    }
+}
+```
+
+### JavaScript Usage Examples
+
+```javascript
+// Streaming
+const stream = await handler.handleRequestStream(request);
+let chunk;
+while ((chunk = await stream.nextChunk()) !== null) {
+    console.log('Received chunk:', chunk);
+}
+
+// WebSocket
+const ws = await handler.handleWebsocket(wsRequest);
+await ws.send('Hello Python!');
+const message = await ws.receive();
+```
+
+### Implementation Notes
+
+1. **Streaming**: The key is to return a handle to the stream rather than waiting for the complete response
+2. **WebSocket**: Requires bidirectional channels and persistent connection state
+3. **Backpressure**: Both implementations should respect backpressure through channel bounds
+4. **Error Handling**: Connection errors need to be propagated through the stream/websocket interface
+
+These features can be added incrementally after the basic HTTP request/response cycle is working correctly with the Python event loop architecture.

fix-python-soname.js

@@ -22,7 +22,7 @@ function findNodeFile() {
 
   // Only run on Linux - other platforms don't need soname fixing
   if (platform !== 'linux') {
-    console.log(`Skipping soname fix on ${platform} - only needed on Linux`);
+    console.log(`fix-python-soname: Skipping soname fix on ${platform} - only needed on Linux`);
     return null;
   }
 
@@ -34,7 +34,7 @@ function findNodeFile() {
 
   const target = archMap[arch];
   if (!target) {
-    console.log(`Unsupported architecture: ${arch}`);
+    console.log(`fix-python-soname: Unsupported architecture: ${arch}`);
     return null;
   }
 
@@ -53,20 +53,29 @@ function findNodeFile() {
       .map(f => path.join(__dirname, f))
   ];
 
-  console.log('Searching for .node files in:');
+  console.log('fix-python-soname: Searching for .node files in:');
   console.log(`  Current directory: ${__dirname}`);
   console.log('  Possible paths:');
   possiblePaths.forEach(p => console.log(`    - ${p}`));
 
   for (const nodePath of possiblePaths) {
     if (fs.existsSync(nodePath)) {
-      console.log(`Found .node file: ${nodePath}`);
+      console.log(`fix-python-soname: Found .node file: ${nodePath}`);
+      // Check if file is readable and has proper permissions
+      try {
+        const stats = fs.statSync(nodePath);
+        console.log(`  File size: ${stats.size} bytes`);
+        console.log(`  File permissions: ${stats.mode.toString(8)}`);
+        console.log(`  File modified: ${stats.mtime}`);
+      } catch (err) {
+        console.error(`  Error getting file stats: ${err.message}`);
+      }
       return nodePath;
     }
   }
 
-  console.log('No .node file found for current platform');
-  console.log('Directory contents:', fs.readdirSync(__dirname).filter(f => !f.startsWith('.')));
+  console.log('fix-python-soname: No .node file found for current platform');
+  console.log('fix-python-soname: Directory contents:', fs.readdirSync(__dirname).filter(f => !f.startsWith('.')));
   return null;
 }
 
@@ -76,17 +85,20 @@ if (!nodeFilePath) {
   process.exit(0); // Exit silently if not applicable
 }
 
-console.log(`Fixing Python soname for: ${nodeFilePath}`);
+console.log(`fix-python-soname: Preparing to fix Python soname for: ${nodeFilePath}`);
 
 // Create a WASI instance
 const wasi = new WASI({
+  version: 'preview1', // Required for newer Node.js versions
   args: ['fix-python-soname', nodeFilePath],
   env: process.env,
   preopens: {
     '/': '/',  // Give access to the filesystem
   }
 });
 
+console.log('fix-python-soname: WASI instance created successfully');
+
 // Load and instantiate the WASM module
 // Try multiple possible locations for the WASM file
 const possibleWasmPaths = [
@@ -103,31 +115,48 @@ for (const tryPath of possibleWasmPaths) {
 }
 
 if (!wasmPath) {
-  console.error('WASM module not found in any of the expected locations:');
+  console.error('fix-python-soname: WASM module not found in any of the expected locations:');
   possibleWasmPaths.forEach(p => console.error(`  - ${p}`));
   process.exit(1);
 }
 
-console.log('Loading WASM soname patcher...');
+console.log(`fix-python-soname: Loading WASM module from: ${wasmPath}`);
 
 async function runSonameFixer() {
   try {
+    console.log('fix-python-soname: Reading WASM file...');
     const wasm = fs.readFileSync(wasmPath);
+    console.log(`fix-python-soname: WASM file loaded, size: ${wasm.length} bytes`);
+    
+    console.log('fix-python-soname: Instantiating WebAssembly module...');
     const { instance } = await WebAssembly.instantiate(wasm, {
       wasi_snapshot_preview1: wasi.wasiImport
     });
+    console.log('fix-python-soname: WebAssembly module instantiated successfully');
 
     // Run the WASI module
+    console.log('fix-python-soname: Starting WASI execution...');
     const exitCode = wasi.start(instance);
+    console.log(`fix-python-soname: WASI execution completed with exit code: ${exitCode}`);
 
     if (exitCode === 0) {
       console.log('✅ Python soname successfully fixed');
+      
+      // Verify the fix by checking if backup was created
+      const backupPath = nodeFilePath.replace('.node', '.node.bak');
+      if (fs.existsSync(backupPath)) {
+        console.log(`fix-python-soname: Backup created at: ${backupPath}`);
+        const originalSize = fs.statSync(backupPath).size;
+        const newSize = fs.statSync(nodeFilePath).size;
+        console.log(`fix-python-soname: Original size: ${originalSize}, New size: ${newSize}`);
+      }
     } else {
       console.error(`❌ Soname fixer exited with code: ${exitCode}`);
       process.exit(exitCode);
     }
   } catch (err) {
     console.error('❌ Error running soname fixer:', err.message);
+    console.error('fix-python-soname: Full error:', err);
 
     // Don't fail the install if soname fixing fails - it's a best-effort operation
     console.log('⚠️  Continuing without soname fix. Python integration may not work correctly.');

fix-python-soname/src/main.rs

@@ -2,6 +2,8 @@ use arwen::elf::ElfContainer;
 use std::{collections::HashMap, env, fs::{self, File}, path::Path};
 
 fn find_python_library() -> Result<String, String> {
+    eprintln!("fix-python-soname: Starting search for Python library...");
+    
     // Generate Python versions from 3.20 down to 3.8
     let mut python_versions = Vec::new();
     for major in (8..=20).rev() {
@@ -15,6 +17,8 @@ fn find_python_library() -> Result<String, String> {
         python_versions.push(format!("libpython3.{}m.so", major));
     }
 
+    eprintln!("fix-python-soname: Looking for versions: {:?}", &python_versions[0..6]);
+    
     // Get system architecture
     let arch = std::env::consts::ARCH;
     let arch_triplet = match arch {
@@ -178,17 +182,22 @@ fn find_python_library() -> Result<String, String> {
         }
     }
 
+    eprintln!("fix-python-soname: Searching in {} directories...", lib_paths.len());
+    eprintln!("fix-python-soname: First 5 paths: {:?}", &lib_paths[0..5.min(lib_paths.len())]);
+    
     // First try exact version matches
     for lib_name in &python_versions {
         for lib_path in &lib_paths {
             let full_path = format!("{}/{}", lib_path, lib_name);
             if Path::new(&full_path).exists() {
-                eprintln!("Found Python library: {}", lib_name);
+                eprintln!("fix-python-soname: Found Python library: {} at {}", lib_name, full_path);
                 return Ok(lib_name.to_string());
             }
         }
     }
 
+    eprintln!("fix-python-soname: No exact match found, searching for any libpython*.so files...");
+    
     // If no exact match found, search directories for any libpython*.so
     for lib_path in &lib_paths {
         if let Ok(entries) = fs::read_dir(lib_path) {
@@ -214,7 +223,7 @@ fn find_python_library() -> Result<String, String> {
             found_libs.sort_by(|a, b| b.2.cmp(&a.2).then(b.1.cmp(&a.1)));
 
             if let Some((lib_name, _, _)) = found_libs.first() {
-                eprintln!("Found Python library: {}", lib_name);
+                eprintln!("fix-python-soname: Found Python library: {} in {}", lib_name, lib_path);
                 return Ok(lib_name.clone());
             }
         }
@@ -226,68 +235,81 @@ fn find_python_library() -> Result<String, String> {
 }
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
+    eprintln!("fix-python-soname: Starting soname patcher...");
+    
     let args: Vec<String> = env::args().collect();
+    eprintln!("fix-python-soname: Arguments: {:?}", args);
 
     if args.len() != 2 {
         return Err(format!("Usage: {} <path-to-node-file>", args[0]).into());
     }
 
     let node_file_path = &args[1];
+    eprintln!("fix-python-soname: Processing file: {}", node_file_path);
 
     // Find the local Python library
     let new_python_lib = find_python_library()?;
 
     // Read the file
+    eprintln!("fix-python-soname: Reading ELF file...");
     let file_contents = fs::read(node_file_path)
         .map_err(|error| format!("Failed to read file: {error}"))?;
+    eprintln!("fix-python-soname: ELF file size: {} bytes", file_contents.len());
 
     // Parse the ELF file
+    eprintln!("fix-python-soname: Parsing ELF file...");
     let mut elf = ElfContainer::parse(&file_contents)
         .map_err(|error| format!("Failed to parse ELF: {error}"))?;
 
     // Get the list of needed libraries
+    eprintln!("fix-python-soname: Getting needed libraries...");
     let needed_libs: Vec<String> = elf.inner.elf_needed()
         .map(|lib| String::from_utf8_lossy(lib).to_string())
         .collect();
 
+    eprintln!("fix-python-soname: Needed libraries: {:?}", needed_libs);
+    
     // Find the existing Python dependency
     let python_lib = needed_libs.iter()
         .find(|lib| lib.starts_with("libpython") && lib.contains(".so"))
         .ok_or("No Python library dependency found in the binary")?;
 
-    eprintln!("Current Python dependency: {python_lib}");
+    eprintln!("fix-python-soname: Current Python dependency: {}", python_lib);
 
     // Check if already pointing to the correct library
     if python_lib == &new_python_lib {
-        eprintln!("Already using the correct Python library");
+        eprintln!("fix-python-soname: Already using the correct Python library");
         return Ok(());
     }
 
-    eprintln!("Replacing with: {new_python_lib}");
+    eprintln!("fix-python-soname: Replacing with: {}", new_python_lib);
 
     // Create a map for replacement
     let mut replacements = HashMap::new();
     replacements.insert(python_lib.clone(), new_python_lib);
 
     // Replace the needed dependency
+    eprintln!("fix-python-soname: Replacing dependency...");
     elf.replace_needed(&replacements)
         .map_err(|error| format!("Failed to replace needed dependency: {error}"))?;
 
     // Create backup
     let file_path = Path::new(node_file_path);
     let backup_path = file_path.with_extension("node.bak");
+    eprintln!("fix-python-soname: Creating backup at: {}", backup_path.display());
     fs::copy(file_path, &backup_path)
         .map_err(|error| format!("Failed to create backup: {error}"))?;
-    eprintln!("Created backup: {}", backup_path.display());
+    eprintln!("fix-python-soname: Backup created successfully");
 
     // Write the modified file
+    eprintln!("fix-python-soname: Writing modified ELF file...");
     let output_file = File::create(node_file_path)
         .map_err(|error| format!("Failed to create output file: {error}"))?;
 
     elf.write(&output_file)
         .map_err(|error| format!("Failed to write ELF: {error}"))?;
 
-    eprintln!("Successfully updated: {node_file_path}");
+    eprintln!("fix-python-soname: Successfully updated: {}", node_file_path);
 
     Ok(())
 }