Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 26 additions & 11 deletions docs-site/scripts/gen_api_docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@
Outputs under docs-site/docs/api/python/modules.
Also writes a coverage and gaps REPORT.md.
"""
import sys
import inspect

import importlib
import inspect
import pkgutil
import sys
from pathlib import Path
from typing import Any, Dict, List, Tuple, Optional
from typing import Any, Dict, List, Optional, Tuple

ROOT = Path(__file__).resolve().parents[2]
PKG_NAME = "nexla_sdk"
Expand All @@ -28,7 +29,9 @@ def iter_module_names(package: str) -> List[str]:
spec = importlib.util.find_spec(package)
if spec is None or not spec.submodule_search_locations:
return names
for m in pkgutil.walk_packages(spec.submodule_search_locations, prefix=f"{package}."):
for m in pkgutil.walk_packages(
spec.submodule_search_locations, prefix=f"{package}."
):
# Skip private or cache
if any(part.startswith("_") for part in m.name.split(".")):
continue
Expand All @@ -50,9 +53,13 @@ def public_members(mod) -> Tuple[List[Tuple[str, Any]], List[Tuple[str, Any]]]:
for n, obj in inspect.getmembers(mod):
if n.startswith("_"):
continue
if inspect.isclass(obj) and getattr(obj, "__module__", "").startswith(mod.__name__):
if inspect.isclass(obj) and getattr(obj, "__module__", "").startswith(
mod.__name__
):
classes.append((n, obj))
elif inspect.isfunction(obj) and getattr(obj, "__module__", "").startswith(mod.__name__):
elif inspect.isfunction(obj) and getattr(obj, "__module__", "").startswith(
mod.__name__
):
functions.append((n, obj))
return classes, functions

Expand Down Expand Up @@ -94,6 +101,7 @@ def pydantic_fields(cls) -> List[Tuple[str, str, Optional[str]]]:
def enum_members(cls) -> List[Tuple[str, Any]]:
try:
import enum

if issubclass(cls, enum.Enum):
return [(m.name, m.value) for m in cls] # type: ignore[attr-defined]
except Exception:
Expand All @@ -108,7 +116,9 @@ def format_signature(obj) -> str:
return "()"


def write_module_page(module_name: str, mod, coverage: Dict[str, Any], gaps: List[str]) -> None:
def write_module_page(
module_name: str, mod, coverage: Dict[str, Any], gaps: List[str]
) -> None:
classes, functions = public_members(mod)
file, line = get_source_info(mod)
title = module_name
Expand All @@ -123,7 +133,7 @@ def write_module_page(module_name: str, mod, coverage: Dict[str, Any], gaps: Lis
TRACE[module_name] = {
"module_source": f"{file}:{line}" if file and line else None,
"classes": {},
"functions": {}
"functions": {},
}

with out_path.open("w", encoding="utf-8") as f:
Expand Down Expand Up @@ -172,7 +182,8 @@ def write_module_page(module_name: str, mod, coverage: Dict[str, Any], gaps: Lis
methods = [
(n, m)
for n, m in inspect.getmembers(cls, predicate=inspect.isfunction)
if not n.startswith("_") and getattr(m, "__module__", "").startswith(mod.__name__)
if not n.startswith("_")
and getattr(m, "__module__", "").startswith(mod.__name__)
]
if methods:
f.write("Methods:\n\n")
Expand All @@ -183,7 +194,9 @@ def write_module_page(module_name: str, mod, coverage: Dict[str, Any], gaps: Lis
f.write(f"- `{n}{sig}`\n")
if mfile and mline:
f.write(f" - Source: `{mfile}:{mline}`\n")
TRACE[module_name]["classes"][f"{cname}.{n}"] = f"{mfile}:{mline}"
TRACE[module_name]["classes"][
f"{cname}.{n}"
] = f"{mfile}:{mline}"
if mdoc:
f.write(f" - {mdoc.splitlines()[0]}\n")
f.write("\n")
Expand All @@ -205,7 +218,9 @@ def write_module_page(module_name: str, mod, coverage: Dict[str, Any], gaps: Lis

# TODO marker if no symbols found
if total_symbols == 0:
f.write("🚧 TODO: No public symbols detected. Verify module visibility and docstrings.\n\n")
f.write(
"🚧 TODO: No public symbols detected. Verify module visibility and docstrings.\n\n"
)
if file and line:
gaps.append(f"No symbols in {file}:{line}")

Expand Down
155 changes: 83 additions & 72 deletions examples/basic_usage.py
Original file line number Diff line number Diff line change
@@ -1,153 +1,164 @@
import os

from nexla_sdk import NexlaClient
from nexla_sdk.models.credentials.requests import CredentialCreate
from nexla_sdk.models.sources.requests import SourceCreate
from nexla_sdk.models.nexsets.requests import NexsetCreate
from nexla_sdk.models.destinations.requests import DestinationCreate
from nexla_sdk.models.nexsets.requests import NexsetCreate
from nexla_sdk.models.sources.requests import SourceCreate


def main():
# Initialize client
client = NexlaClient(
service_key=os.getenv("NEXLA_SERVICE_KEY"),
base_url=os.getenv("NEXLA_API_URL", "https://dataops.nexla.io/nexla-api")
base_url=os.getenv("NEXLA_API_URL", "https://dataops.nexla.io/nexla-api"),
)

# Example 1: List all sources
print("=== Listing Sources ===")
sources = client.sources.list()
for source in sources:
print(f"Source: {source.name} ({source.id}) - Status: {source.status}")

# Example 2: Create a credential
print("\n=== Creating Credential ===")
credential = client.credentials.create(CredentialCreate(
name="My S3 Bucket",
credentials_type="s3",
credentials={
"access_key_id": "your_access_key",
"secret_access_key": "your_secret_key",
"region": "us-east-1"
}
))
credential = client.credentials.create(
CredentialCreate(
name="My S3 Bucket",
credentials_type="s3",
credentials={
"access_key_id": "your_access_key",
"secret_access_key": "your_secret_key",
"region": "us-east-1",
},
)
)
print(f"Created credential: {credential.name} ({credential.id})")

# Example 3: Create a source
print("\n=== Creating Source ===")
source = client.sources.create(SourceCreate(
name="My S3 Source",
source_type="s3",
data_credentials_id=credential.id,
source_config={
"path": "my-bucket/data/",
"file_format": "json",
"start.cron": "0 0 * * * ?" # Daily at midnight
}
))
source = client.sources.create(
SourceCreate(
name="My S3 Source",
source_type="s3",
data_credentials_id=credential.id,
source_config={
"path": "my-bucket/data/",
"file_format": "json",
"start.cron": "0 0 * * * ?", # Daily at midnight
},
)
)
print(f"Created source: {source.name} ({source.id})")

# Example 4: Get detected nexsets
print("\n=== Detected Nexsets ===")
nexsets = client.nexsets.list()
source_nexsets = [n for n in nexsets if n.data_source_id == source.id]
for nexset in source_nexsets:
print(f"Nexset: {nexset.name} ({nexset.id})")

# Example 5: Create a transformed nexset
if source_nexsets:
parent_nexset = source_nexsets[0]
print(f"\n=== Creating Transformed Nexset from {parent_nexset.name} ===")

transformed = client.nexsets.create(NexsetCreate(
name="Transformed Data",
parent_data_set_id=parent_nexset.id,
has_custom_transform=True,
transform={
"version": 1,
"operations": [{
"operation": "shift",
"spec": {
"*": "&" # Pass through all fields
}
}]
}
))

transformed = client.nexsets.create(
NexsetCreate(
name="Transformed Data",
parent_data_set_id=parent_nexset.id,
has_custom_transform=True,
transform={
"version": 1,
"operations": [
{
"operation": "shift",
"spec": {"*": "&"}, # Pass through all fields
}
],
},
)
)
print(f"Created nexset: {transformed.name} ({transformed.id})")

# Example 6: Create a destination
print("\n=== Creating Destination ===")
destination = client.destinations.create(DestinationCreate(
name="My S3 Output",
sink_type="s3",
data_credentials_id=credential.id,
data_set_id=transformed.id,
sink_config={
"path": "my-bucket/output/",
"file_format": "parquet",
"file_compression": "snappy"
}
))
destination = client.destinations.create(
DestinationCreate(
name="My S3 Output",
sink_type="s3",
data_credentials_id=credential.id,
data_set_id=transformed.id,
sink_config={
"path": "my-bucket/output/",
"file_format": "parquet",
"file_compression": "snappy",
},
)
)
print(f"Created destination: {destination.name} ({destination.id})")

# Example 7: View flow
print("\n=== Flow Structure ===")
flows = client.flows.list(flows_only=True)
if flows:
flow = flows[0]
print(f"Flow has {len(flow.flows)} nodes")

# Example 8: Pagination example
print("\n=== Pagination Example ===")
paginator = client.sources.paginate(per_page=10)

# Iterate through all items
for source in paginator:
print(f"Source: {source.name}")

# Or iterate by pages
for page in paginator.iter_pages():
print(f"Page {page.page_info.current_page}: {len(page.items)} items")

# Example 9: Error handling
print("\n=== Error Handling Example ===")
try:
client.sources.get(999999) # Non-existent ID
except Exception as e:
print(f"Expected error: {type(e).__name__}: {e}")

# Example 10: Access control
print("\n=== Access Control Example ===")
if source_nexsets:
nexset = source_nexsets[0]

# Get current accessors
accessors = client.nexsets.get_accessors(nexset.id)
print(f"Current accessors: {len(accessors)}")

# Add a user accessor
new_accessors = [{
"type": "USER",
"email": "colleague@example.com",
"access_roles": ["collaborator"]
}]

new_accessors = [
{
"type": "USER",
"email": "colleague@example.com",
"access_roles": ["collaborator"],
}
]

updated = client.nexsets.add_accessors(nexset.id, new_accessors)
print(f"Updated accessors: {len(updated)}")

# Example 11: Metrics
print("\n=== Metrics Example ===")
from datetime import datetime, timedelta

yesterday = (datetime.now() - timedelta(days=1)).strftime("%Y-%m-%d")
today = datetime.now().strftime("%Y-%m-%d")

if sources:
source = sources[0]
metrics = client.metrics.get_resource_daily_metrics(
resource_type="data_sources",
resource_id=source.id,
from_date=yesterday,
to_date=today
to_date=today,
)
print(f"Metrics status: {metrics.status}")
if metrics.metrics:
Expand Down
Loading
Loading