updates (#7)

updates

Author: bauman

.ci/generate_fake_data.py

@@ -4,25 +4,34 @@
 
 fake = Faker()
 
-
-print("generate 5000 fake profiles, write them all into a bson object,"
-      "read back only those with @gmail.com emails")
-
-
-with open("fake_profiles.bson", "wb") as f:
-    for _ in range(500):
-        faked = fake.simple_profile()
-        del faked['birthdate']  # bson doesn't like date wants datetime
-        bson_data = bson.BSON.encode(faked)
-        f.write(bson_data)
-
-found_gmails = 0
-with open("fake_profiles.bson", "rb") as f:
-    stream = BSONInput(fh=f, fast_string_prematch=b"@gmail.com")
-    for doc in stream:
-        assert "@gmail" in doc['mail']  # bson handles the utf8 decoding by default!
-        found_gmails += 1
-
-
-assert found_gmails > 0
-print(f"found {found_gmails} from gmails")
+if __name__ == "__main__":
+    print("generate 5000 fake profiles, write them all into a bson object,"
+          "read back only those with @gmail.com emails")
+
+
+    with open("fake_profiles.bson", "wb") as f:
+        for _ in range(500):
+            faked = fake.simple_profile()
+            del faked['birthdate']  # bson doesn't like date wants datetime
+            bson_data = bson.encode(faked)
+            f.write(bson_data)
+
+    found_gmails = 0
+    with open("fake_profiles.bson", "rb") as f:
+        stream = BSONInput(fh=f, fast_string_prematch=b"@gmail.com")
+        for doc in stream:
+            assert "@gmail" in doc['mail']  # bson handles the utf8 decoding by default!
+            found_gmails += 1
+
+
+    assert found_gmails > 0
+    print(f"found {found_gmails} from gmails")
+
+    found_gmails_raw = 0
+    with open("fake_profiles.bson", "rb") as f:
+        stream = BSONInput(fh=f, fast_string_prematch=b"@gmail.com", decode=False)
+        for raw_bson in stream:
+            assert b"@gmail" in raw_bson  # not even bothering to decode to a dict
+            found_gmails_raw += 1
+    assert found_gmails_raw > 0
+    print(f"found {found_gmails_raw} from gmails without even decoding the BSON")

.github/workflows/python.yml .github/workflows/python-test.yml.github/workflows/python.yml renamed to .github/workflows/python-test.yml

@@ -1,27 +1,31 @@
-name: Python package
+name: test the module
 
 on:
   push:
-    branches: [ master ]
+    branches: [ master, dev ]
   pull_request:
     branches: [ master ]
 
 jobs:
   simpletest:
     runs-on:
-    - ubuntu-18.04
+    - ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8]
+        python-version: [3.11, 3.12, 3.13, 3.14]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v5
+
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
+
+
     - name: run a simple test
       run: |
-        pip install pymongo Faker
+        pip install pymongo faker
         cp .ci/generate_fake_data.py .
-        python generate_fake_data.py
+        python3 generate_fake_data.py
+

.github/workflows/python-wheel.yml

@@ -0,0 +1,26 @@
+name: build the wheel
+
+on:
+  push:
+    branches: [ master, dev ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  wheelbuild:
+    runs-on:
+    - ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v5
+
+
+    - name: build and run a simple test
+      run: |
+        pip install wheel build pymongo faker
+        python3 -m build
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: wheel-file
+        path: ./dist/*.whl

.gitignore

@@ -1,3 +1,7 @@
+#test data
+fake_profiles.bson
+
+
 #################
 ## Eclipse
 #################

README.md

@@ -42,7 +42,7 @@ The fast_string_prematch would not bother converting records that do not have "g
 somewhere in the document as plaintext.
 
 ``` python
- from bsonstream import KeyValueBSONInput
+ from bsonstream import BSONInput
  from sys import argv
  import gzip
  for file in argv[1:]:
@@ -51,12 +51,30 @@ somewhere in the document as plaintext.
          f = open(file, 'rb')
      else:
          f=gzip.open(file,'rb')
-     stream = KeyValueBSONInput(fh=f,  fast_string_prematch=b"github")
+     stream = BSONInput(fh=f,  fast_string_prematch=b"github")
      for dict_data in stream:
          ...process dict_data...
 ```
 
 
+or if you are passing data to another tool that can handle raw bson (like bsonsearch), don't even bother decoding the BSON to a dict
+
+``` python
+ from bsonstream import BSONInput
+ from sys import argv
+ import gzip
+ for file in argv[1:]:
+     f=None
+     if "gz" not in file:
+         f = open(file, 'rb')
+     else:
+         f=gzip.open(file,'rb')
+     stream = BSONInput(fh=f,  fast_string_prematch=b"github")
+     for raw_bson in stream:
+         ...process dict_data...
+```
+
+
 ## Benchmark
 Unfortunately, I cannot make available the test bson file.
 
@@ -85,7 +103,7 @@ With fast string matcher.  In this case, documents matching the fast string pate
 ## Dependencies
 
 Required libraries
-* [python-bson] 
+* [pymongo] 
 
 
 ## Versioning

pyproject.toml

@@ -0,0 +1,31 @@
+[project]
+name = "bsonstream"
+version = "0.1.7"
+description = "BSON stream raw data into dict or individual BSON format - python"
+
+requires-python = ">=3.11"
+dependencies = [
+    "pymongo~=4.15.2"
+]
+
+[project.urls]
+GitHub = "https://github.com/bauman/python-bson-streaming"
+
+[build-system]
+requires = ["setuptools>=61"]
+build-backend = "setuptools.build_meta"
+
+
+[tool.cibuildwheel.linux]
+# This command runs for the manylinux containers (based on CentOS).
+archs = ["x86_64"]
+
+
+[tool.cibuildwheel.macos]
+archs = ["x86_64", "universal2", "arm64"]
+
+
+[[tool.cibuildwheel.overrides]]
+select = "*-musllinux*"
+# This command runs for the musllinux containers (based on Alpine Linux).
+archs = ["x86_64"]