Skip to content

Calling CSharp releases the GIL #7970

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Martin-Molinero merged 1 commit into from
Apr 23, 2024
Merged

Calling CSharp releases the GIL #7970

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Algorithm.CSharp/QuantConnect.Algorithm.CSharp.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
<DebugType>portable</DebugType>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.35" />
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.36" />
<PackageReference Include="Accord" Version="3.6.0" />
<PackageReference Include="Accord.Fuzzy" Version="3.6.0" />
<PackageReference Include="Accord.MachineLearning" Version="3.6.0" />
Expand Down
2 changes: 1 addition & 1 deletion Algorithm.Framework/QuantConnect.Algorithm.Framework.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
<PackageLicenseFile>LICENSE</PackageLicenseFile>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.35" />
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.36" />
<PackageReference Include="Accord" Version="3.6.0" />
<PackageReference Include="Accord.Math" Version="3.6.0" />
<PackageReference Include="Accord.Statistics" Version="3.6.0" />
Expand Down
2 changes: 1 addition & 1 deletion Algorithm.Python/QuantConnect.Algorithm.Python.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
<Compile Include="..\Common\Properties\SharedAssemblyInfo.cs" Link="Properties\SharedAssemblyInfo.cs" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.35" />
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.36" />
</ItemGroup>
<ItemGroup>
<Content Include="FundamentalUniverseSelectionAlgorithm.py" />
Expand Down
2 changes: 1 addition & 1 deletion Algorithm/QuantConnect.Algorithm.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
<PackageLicenseFile>LICENSE</PackageLicenseFile>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.35" />
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.36" />
<PackageReference Include="MathNet.Numerics" Version="5.0.0" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
<PackageReference Include="NodaTime" Version="3.0.5" />
Expand Down
2 changes: 1 addition & 1 deletion AlgorithmFactory/QuantConnect.AlgorithmFactory.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
<PackageLicenseFile>LICENSE</PackageLicenseFile>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.35" />
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.36" />
<PackageReference Include="NodaTime" Version="3.0.5" />
</ItemGroup>
<ItemGroup>
Expand Down
2 changes: 1 addition & 1 deletion Common/QuantConnect.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
<Message Text="SelectedOptimization $(SelectedOptimization)" Importance="high" />
</Target>
<ItemGroup>
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.35" />
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.36" />
<PackageReference Include="CloneExtensions" Version="1.3.0" />
<PackageReference Include="fasterflect" Version="3.0.0" />
<PackageReference Include="MathNet.Numerics" Version="5.0.0" />
Expand Down
2 changes: 1 addition & 1 deletion Engine/QuantConnect.Lean.Engine.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
<Message Text="SelectedOptimization $(SelectedOptimization)" Importance="high" />
</Target>
<ItemGroup>
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.35" />
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.36" />
<PackageReference Include="fasterflect" Version="3.0.0" />
<PackageReference Include="MathNet.Numerics" Version="5.0.0" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
Expand Down
2 changes: 1 addition & 1 deletion Indicators/QuantConnect.Indicators.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
<Message Text="SelectedOptimization $(SelectedOptimization)" Importance="high" />
</Target>
<ItemGroup>
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.35" />
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.36" />
<PackageReference Include="MathNet.Numerics" Version="5.0.0" />
</ItemGroup>
<ItemGroup>
Expand Down
2 changes: 1 addition & 1 deletion Report/QuantConnect.Report.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
<PackageLicenseFile>LICENSE</PackageLicenseFile>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.35" />
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.36" />
<PackageReference Include="Deedle" Version="2.1.0" />
<PackageReference Include="MathNet.Numerics" Version="5.0.0" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
Expand Down
2 changes: 1 addition & 1 deletion Research/QuantConnect.Research.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
<ItemGroup>
<PackageReference Include="Plotly.NET" Version="3.0.1" />
<PackageReference Include="Plotly.NET.Interactive" Version="3.0.2" />
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.35" />
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.36" />
<PackageReference Include="NodaTime" Version="3.0.5" />
</ItemGroup>
<ItemGroup>
Expand Down
105 changes: 102 additions & 3 deletions Tests/Python/PythonThreadingTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/*
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
Expand All @@ -14,16 +14,99 @@
*
*/

using System.Threading.Tasks;
using NUnit.Framework;
using System;
using Python.Runtime;
using NUnit.Framework;
using System.Threading;
using QuantConnect.Python;
using System.Threading.Tasks;

namespace QuantConnect.Tests.Python
{
[TestFixture]
public class PythonThreadingTests
{
[TestCase("Field", false)]
[TestCase("Func()", false)]
[TestCase("Property", false)]
[TestCase("Field", true)]
[TestCase("Func()", true)]
[TestCase("Property", true)]
public void CallingCShapReleasesGil(string target, bool useInstance)
{
var lockInstance = new object();

using var tookGil = new ManualResetEvent(false);
using var tookLock = new ManualResetEvent(false);

PyObject method;
PyObject propertyWrapper;
using (Py.GIL())
{
var module = PyModule.FromString("ReleaseGil", $@"
from clr import AddReference
AddReference('QuantConnect.Tests')

from QuantConnect.Tests.Python import *
import time

def Method():
return 1

def PropertyCaller(tookGil, tookLock, useInstance):
tookGil.Set()
tookLock.WaitOne()
if useInstance:
instance = PythonThreadingTests.TestPropertyWrapper()
return instance.Instance{target}
else:
return PythonThreadingTests.TestPropertyWrapper.{target}
");
method = module.GetAttr("Method");
propertyWrapper = module.GetAttr("PropertyCaller");
}

TestPropertyWrapper.Func = () =>
{
lock (lockInstance)
{
Thread.Sleep(500);
using (Py.GIL())
{
return method.Invoke().As<int>();
}
}
};

// task1: has the GIL, go into python, go back into C# and want the C# lock, so he should release the GIL
// task2: has the C# lock and want's the GIL
var task1 = Task.Run(() =>
{
using (Py.GIL())
{
propertyWrapper.Invoke(tookGil, tookLock, useInstance);
}
});

var task2 = Task.Run(() =>
{
lock (lockInstance)
{
// we take the C# lock and wait and try to get the py gil which should be taken by task 1
tookLock.Set();
tookGil.WaitOne();
using (Py.GIL())
{
method.Invoke();
}
}
});

var result = Task.WaitAll(new[] { task1, task2 }, TimeSpan.FromSeconds(3));

Assert.IsTrue(result);
}

[Test]
public void ImportsCanBeExecutedFromDifferentThreads()
{
Expand All @@ -48,5 +131,21 @@ public void ImportsCanBeExecutedFromDifferentThreads()
}
}).Wait();
}
public class TestPropertyWrapper
{
public static Func<int> Func { get; set; }
public static int Field => Func();
public static int Property
{
get
{
return Func();
}
}

public Func<int> InstanceFunc => Func;
public int InstanceField => Field;
public int InstanceProperty => Property;
}
}
}
2 changes: 1 addition & 1 deletion Tests/QuantConnect.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
</PropertyGroup>
<Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
<ItemGroup>
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.35" />
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.36" />
<PackageReference Include="Accord" Version="3.6.0" />
<PackageReference Include="Accord.Math" Version="3.6.0" />
<PackageReference Include="Common.Logging" Version="3.4.1" />
Expand Down