From 08e03603d1d1b03249c4f20572c00fe3036c4ed1 Mon Sep 17 00:00:00 2001 From: elijahpetty Date: Fri, 23 Jan 2026 13:51:15 -0600 Subject: [PATCH 01/13] . --- README.md | 325 +++++++++++------- data/batch/file1.csv | 4 + data/batch/file2.csv | 4 + data/batch/file3.csv | 5 + data/sample.csv | 11 + my_dh_cli/README.md | 57 +++ my_dh_cli/pyproject.toml | 20 ++ my_dh_cli/src/my_dh_cli.egg-info/PKG-INFO | 59 ++++ my_dh_cli/src/my_dh_cli.egg-info/SOURCES.txt | 11 + .../my_dh_cli.egg-info/dependency_links.txt | 1 + .../src/my_dh_cli.egg-info/entry_points.txt | 2 + my_dh_cli/src/my_dh_cli.egg-info/requires.txt | 2 + .../src/my_dh_cli.egg-info/top_level.txt | 1 + my_dh_cli/src/my_dh_cli/__init__.py | 3 + my_dh_cli/src/my_dh_cli/__main__.py | 4 + .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 267 bytes .../my_dh_cli/__pycache__/cli.cpython-312.pyc | Bin 0 -> 1606 bytes my_dh_cli/src/my_dh_cli/cli.py | 31 ++ my_dh_library/README.md | 55 +++ my_dh_library/pyproject.toml | 16 + .../src/my_dh_library.egg-info/PKG-INFO | 55 +++ .../src/my_dh_library.egg-info/SOURCES.txt | 10 + .../dependency_links.txt | 1 + .../src/my_dh_library.egg-info/requires.txt | 1 + .../src/my_dh_library.egg-info/top_level.txt | 1 + my_dh_library/src/my_dh_library/__init__.py | 7 + .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 427 bytes .../__pycache__/queries.cpython-312.pyc | Bin 0 -> 1355 bytes my_dh_library/src/my_dh_library/queries.py | 30 ++ my_dh_library/src/my_dh_library/utils.py | 18 + my_dh_toolkit/README.md | 104 ++++++ my_dh_toolkit/pyproject.toml | 21 ++ my_dh_toolkit/src/my_dh_toolkit/__init__.py | 8 + my_dh_toolkit/src/my_dh_toolkit/__main__.py | 4 + my_dh_toolkit/src/my_dh_toolkit/cli.py | 31 ++ my_dh_toolkit/src/my_dh_toolkit/processor.py | 40 +++ my_dh_toolkit/src/my_dh_toolkit/queries.py | 30 ++ my_dh_toolkit/src/my_dh_toolkit/utils.py | 18 + 38 files changed, 871 insertions(+), 119 deletions(-) create mode 100644 data/batch/file1.csv create mode 100644 data/batch/file2.csv create mode 100644 data/batch/file3.csv create mode 100644 data/sample.csv create mode 100644 my_dh_cli/README.md create mode 100644 my_dh_cli/pyproject.toml create mode 100644 my_dh_cli/src/my_dh_cli.egg-info/PKG-INFO create mode 100644 my_dh_cli/src/my_dh_cli.egg-info/SOURCES.txt create mode 100644 my_dh_cli/src/my_dh_cli.egg-info/dependency_links.txt create mode 100644 my_dh_cli/src/my_dh_cli.egg-info/entry_points.txt create mode 100644 my_dh_cli/src/my_dh_cli.egg-info/requires.txt create mode 100644 my_dh_cli/src/my_dh_cli.egg-info/top_level.txt create mode 100644 my_dh_cli/src/my_dh_cli/__init__.py create mode 100644 my_dh_cli/src/my_dh_cli/__main__.py create mode 100644 my_dh_cli/src/my_dh_cli/__pycache__/__init__.cpython-312.pyc create mode 100644 my_dh_cli/src/my_dh_cli/__pycache__/cli.cpython-312.pyc create mode 100644 my_dh_cli/src/my_dh_cli/cli.py create mode 100644 my_dh_library/README.md create mode 100644 my_dh_library/pyproject.toml create mode 100644 my_dh_library/src/my_dh_library.egg-info/PKG-INFO create mode 100644 my_dh_library/src/my_dh_library.egg-info/SOURCES.txt create mode 100644 my_dh_library/src/my_dh_library.egg-info/dependency_links.txt create mode 100644 my_dh_library/src/my_dh_library.egg-info/requires.txt create mode 100644 my_dh_library/src/my_dh_library.egg-info/top_level.txt create mode 100644 my_dh_library/src/my_dh_library/__init__.py create mode 100644 my_dh_library/src/my_dh_library/__pycache__/__init__.cpython-312.pyc create mode 100644 my_dh_library/src/my_dh_library/__pycache__/queries.cpython-312.pyc create mode 100644 my_dh_library/src/my_dh_library/queries.py create mode 100644 my_dh_library/src/my_dh_library/utils.py create mode 100644 my_dh_toolkit/README.md create mode 100644 my_dh_toolkit/pyproject.toml create mode 100644 my_dh_toolkit/src/my_dh_toolkit/__init__.py create mode 100644 my_dh_toolkit/src/my_dh_toolkit/__main__.py create mode 100644 my_dh_toolkit/src/my_dh_toolkit/cli.py create mode 100644 my_dh_toolkit/src/my_dh_toolkit/processor.py create mode 100644 my_dh_toolkit/src/my_dh_toolkit/queries.py create mode 100644 my_dh_toolkit/src/my_dh_toolkit/utils.py diff --git a/README.md b/README.md index c12bb06..dbaf300 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,31 @@ # Python Packaging with Deephaven -This example demonstrates how to create and deploy Python packages that use Deephaven. It shows you how to package both command-line tools and reusable libraries using modern Python packaging standards. +This repository demonstrates how to create and deploy Python packages that use Deephaven. It shows three complete packaging scenarios following the official [Python Packaging User Guide](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/) recommendations. This example accompanies the [Packaging custom code and dependencies](https://deephaven.io/core/docs/how-to-guides/sysadmin/setuptools-deployment/) guide in the Deephaven documentation. ## What you'll learn -This example shows you how to: - - Create installable Python packages with Deephaven dependencies -- Build command-line tools that process data with Deephaven - Package reusable library code for other projects +- Build command-line tools with entry point scripts - Manage dependencies with `pyproject.toml` +- Use the src-layout structure - Distribute packages as wheel archives -## Project structure +## Prerequisites + +- Python 3.8 or later +- pip (Python package installer) +- Basic familiarity with Python packaging + +## Repository structure -The example includes three complete packaging scenarios: +This repository contains three complete packaging scenarios: ### 1. Library-only package (`my_dh_library/`) -A reusable library with Deephaven query functions that other projects can import. +Reusable library code without CLI tools. Other projects import your modules. ``` my_dh_library/ @@ -33,68 +38,74 @@ my_dh_library/ └── README.md ``` +**Usage:** +```python +from my_dh_library.queries import filter_by_threshold +``` + ### 2. CLI-only package (`my_dh_cli/`) -Command-line tools for processing data with Deephaven. +Command-line tool without exposing library code. ``` my_dh_cli/ ├── src/ -│ └── my_dh_package/ +│ └── my_dh_cli/ │ ├── __init__.py │ ├── __main__.py -│ ├── cli.py -│ └── processor.py +│ └── cli.py ├── pyproject.toml -├── data/ -│ └── sample.csv └── README.md ``` +**Usage:** +```python +# Use within a Python session with server running +from my_dh_cli.cli import my_dh_query +result = my_dh_query("input_data.csv", verbose=True) +``` + ### 3. Combined package (`my_dh_toolkit/`) -Both reusable library code and command-line tools in one package. +Both reusable library code and command-line tools. ``` my_dh_toolkit/ ├── src/ -│ └── my_dh_package/ +│ └── my_dh_toolkit/ │ ├── __init__.py │ ├── __main__.py │ ├── cli.py +│ ├── processor.py │ ├── queries.py │ └── utils.py ├── pyproject.toml └── README.md ``` -## Prerequisites +**Usage:** +```python +# As a library +from my_dh_toolkit.queries import filter_by_threshold +``` -- Python 3.8 or later -- pip (Python package installer) -- Basic familiarity with Python packaging +```python +# As CLI functions (within Python session) +from my_dh_toolkit import my_dh_query, batch_process +result = my_dh_query("input_data.csv", verbose=True) +batch_process("data/", "results/", verbose=True) +``` ## Quick start Clone the repository: ```shell -git clone https://github.com/deephaven-examples/python-packaging.git -cd python-packaging +git clone https://github.com/deephaven-examples/deephaven-python-packaging.git +cd deephaven-python-packaging ``` -Choose an example to try: - -### Try the CLI package - -```shell -cd my_dh_cli -pip install -e . -my-dh-query data/sample.csv --verbose -my-dh-process data/ --output results/ -``` - -### Try the library package +### Try the library-only package ```shell cd my_dh_library @@ -105,173 +116,249 @@ python Then in Python: ```python +# Start the Deephaven server +from deephaven_server import Server +server = Server(port=10000, jvm_args=["-Xmx4g"]) +server.start() + +# Now use the library functions from my_dh_library.queries import filter_by_threshold from deephaven import read_csv data = read_csv("../data/sample.csv") filtered = filter_by_threshold(data, "Score", 75.0) -print(f"Filtered to {filtered.size} rows") +``` + +### Try the CLI-only package + +> [!NOTE] +> CLI tools require a Deephaven server running in the same Python process. The examples below show how to use the CLI functions within a Python session where the server is already started. True standalone CLI commands (run from a separate terminal) are not practical with Deephaven due to JVM initialization requirements. + +```shell +cd my_dh_cli +pip install -e . +python +``` + +Then in Python: + +```python +# Start the Deephaven server +from deephaven_server import Server +server = Server(port=10000, jvm_args=["-Xmx4g"]) +server.start() + +# Now use the CLI function +from my_dh_cli.cli import my_dh_query +result = my_dh_query("../data/sample.csv", verbose=True) +print(f"Processed {result.size} rows") ``` ### Try the combined package +> [!NOTE] +> Like the CLI-only package, the CLI commands require a Deephaven server in the same Python process. Use the library functions within a Python session. + ```shell cd my_dh_toolkit pip install -e . +python +``` + +Then in Python: + +```python +# Start the Deephaven server +from deephaven_server import Server +server = Server(port=10000, jvm_args=["-Xmx4g"]) +server.start() # Use as a library -python -c "from my_dh_toolkit.queries import filter_by_threshold; print('Library imported successfully')" +from my_dh_toolkit.queries import filter_by_threshold +from deephaven import read_csv -# Use as CLI tools -my-dh-query ../data/sample.csv -my-dh-process ../data/ --output results/ +data = read_csv("../data/sample.csv") +filtered = filter_by_threshold(data, "Score", 75.0) + +# Or use the CLI functions +from my_dh_toolkit import my_dh_query, batch_process +result = my_dh_query("../data/sample.csv", verbose=True) +batch_process("../data/batch/", "./output", verbose=True) ``` -## What's included +## Sample data -### Command-line tools +The `data/` directory contains sample CSV files for testing: -The CLI examples demonstrate: +- `sample.csv` - Single file with Name, Score, Value, and Category columns +- `batch/` - Multiple CSV files for batch processing examples -- **Entry point scripts** - Commands installed to your PATH -- **Module execution** - Running with `python -m package_name` -- **Argument parsing** - Using Click for robust CLI interfaces -- **Multiple commands** - Single package with multiple tools -- **Verbose output** - Optional detailed logging +## Key concepts + +### Package structure -### Library modules +All examples use the **src-layout**, which is the recommended structure for Python packages: -The library examples show: +``` +my_project/ +├── src/ +│ └── my_package/ +│ ├── __init__.py +│ └── module.py +├── pyproject.toml +└── README.md +``` -- **Reusable query functions** - Common Deephaven operations -- **Type hints** - Proper function signatures -- **Public API exports** - Clean import patterns -- **Documentation** - Docstrings for all functions +The src-layout keeps source code separate from tests and configuration files. -### Configuration +### Entry point scripts -All examples include: +Entry point scripts are defined in `[project.scripts]` and become available after installation: -- **`pyproject.toml`** - Modern Python packaging configuration -- **Dependency management** - Automatic installation of Deephaven and other requirements -- **Version constraints** - Ensuring compatible package versions -- **Entry points** - Mapping command names to Python functions +```toml +[project.scripts] +my-command = "my_package.module:function" +``` -## Building and distributing +After `pip install`, you can run `my-command` from anywhere. -Each example can be built into a distributable wheel: +### Module execution -```shell -cd my_dh_cli # or any example directory -pip install build -python -m build -``` +Add a `__main__.py` file to support running packages with `python -m`: -This creates a `.whl` file in the `dist/` directory that can be: +```python +from my_package.cli import app -- Installed locally: `pip install dist/my_dh_cli-0.1.0-py3-none-any.whl` -- Distributed to others -- Published to PyPI: `python -m twine upload dist/*` +if __name__ == "__main__": + app() +``` -## Running the examples +This allows running without installation: `python -m my_package` -### Development mode +### Dependencies -Install in editable mode to make changes without reinstalling: +Dependencies are specified in `pyproject.toml`: -```shell -pip install -e . +```toml +[project] +dependencies = [ + "deephaven-server>=0.35.0", + "click>=8.0.0", +] ``` -### Regular installation +These are automatically installed when users install your package. -Install from the built wheel: +## Building and distributing + +Build a distributable wheel: ```shell -pip install dist/package_name-0.1.0-py3-none-any.whl +cd my_dh_cli # or any package directory +pip install build +python -m build ``` -### Without installation +This creates a `.whl` file in `dist/` that can be: -Run directly from source using module execution: +- Installed locally: `pip install dist/my_dh_cli-0.1.0-py3-none-any.whl` +- Distributed to others +- Published to PyPI: `python -m twine upload dist/*` -```shell -python -m my_dh_package input_data.csv -``` +## Packaging scenarios -## Sample data +### When to use library-only -The `data/` directory contains sample CSV files for testing: +- Creating reusable code for other projects +- No command-line interface needed +- Code will be imported, not executed -- `sample.csv` - Small dataset with Name, Age, and Score columns -- `batch/` - Multiple CSV files for batch processing examples +**Example:** Data processing utilities, query functions, helper classes -You can use your own CSV files with these examples. +### When to use CLI-only -## Key concepts +- Building command-line tools for end users +- No library code to expose +- Want clean command names -### Entry point scripts vs module execution +**Example:** Data conversion tools, file processors, automation scripts -The examples demonstrate two ways to run Python packages: +### When to use combined -1. **Entry point scripts** - Commands defined in `[project.scripts]` that become available after installation - ```shell - my-dh-query data.csv - ``` +- Need both library and CLI functionality +- Want to provide multiple interfaces +- Library functions useful on their own -2. **Module execution** - Running packages with `python -m` without installation - ```shell - python -m my_dh_package data.csv - ``` +**Example:** Data analysis toolkit with both API and CLI -See the [Execution patterns](https://deephaven.io/core/docs/how-to-guides/sysadmin/setuptools-deployment/#execution-patterns) section of the guide for when to use each method. +## Execution patterns -### Package structure +### Entry point scripts (recommended for CLI tools) -All examples use the **src-layout**, which is the recommended structure for Python packages. This keeps source code separate from tests and configuration files. +**Configure in `pyproject.toml`:** +```toml +[project.scripts] +my-command = "my_package.module:function" +``` -### Dependencies +**Run after installation:** +```shell +my-command +``` -The examples show how to: +**Benefits:** +- Clean command names +- Available system-wide +- Standard Python packaging approach -- Specify required packages (like `deephaven-server`) -- Set version constraints -- Define optional dependencies for features like visualization or testing +### Module execution (useful for development) -## Related documentation +**Add `__main__.py`:** +```python +from my_package.cli import app -- [Packaging custom code and dependencies](https://deephaven.io/core/docs/how-to-guides/sysadmin/setuptools-deployment/) - Complete guide -- [Install and use Python packages](https://deephaven.io/core/docs/how-to-guides/install-and-use-python-packages/) -- [Use the Deephaven Python package](https://deephaven.io/core/docs/how-to-guides/deephaven-python-package/) -- [Python Packaging User Guide](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/) -- [Click documentation](https://click.palletsprojects.com/) +if __name__ == "__main__": + app() +``` + +**Run without installation:** +```shell +python -m my_package +``` + +**Benefits:** +- No installation required +- Useful for development and testing +- Works from source directory ## Troubleshooting ### Command not found after installation -If your command isn't found after installation: - -- Ensure the installation completed without errors +- Ensure installation completed without errors - Check that the installation directory is in your PATH - Try reinstalling: `pip install --force-reinstall .` ### Import errors -If you encounter import errors: - - Verify all dependencies are installed: `pip list` - Check that you're using Python 3.8 or later - Ensure Deephaven is installed: `pip install deephaven-server` ### Module not found errors -If Python can't find your modules: - - Verify `__init__.py` files exist in all package directories - Check that package names in `[project.scripts]` match your directory structure - Try reinstalling in editable mode: `pip install -e .` +## Related documentation + +- [Packaging custom code and dependencies](https://deephaven.io/core/docs/how-to-guides/sysadmin/setuptools-deployment/) - Complete guide +- [Install and use Python packages](https://deephaven.io/core/docs/how-to-guides/install-and-use-python-packages/) +- [Use the Deephaven Python package](https://deephaven.io/core/docs/how-to-guides/deephaven-python-package/) +- [Python Packaging User Guide](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/) +- [Click documentation](https://click.palletsprojects.com/) + ## Note The code in this repository is built for Deephaven Community Core v0.35.0 or later. For the latest Deephaven version, see [deephaven.io](https://deephaven.io/). diff --git a/data/batch/file1.csv b/data/batch/file1.csv new file mode 100644 index 0000000..fe9fae6 --- /dev/null +++ b/data/batch/file1.csv @@ -0,0 +1,4 @@ +Name,Score,Value,Category +Alice,85,120,A +Bob,92,150,B +Charlie,78,95,A diff --git a/data/batch/file2.csv b/data/batch/file2.csv new file mode 100644 index 0000000..e270a59 --- /dev/null +++ b/data/batch/file2.csv @@ -0,0 +1,4 @@ +Name,Score,Value,Category +Diana,88,110,C +Eve,95,180,B +Frank,72,85,A diff --git a/data/batch/file3.csv b/data/batch/file3.csv new file mode 100644 index 0000000..692c4a2 --- /dev/null +++ b/data/batch/file3.csv @@ -0,0 +1,5 @@ +Name,Score,Value,Category +Grace,91,160,C +Henry,83,105,B +Iris,89,140,A +Jack,76,90,C diff --git a/data/sample.csv b/data/sample.csv new file mode 100644 index 0000000..1d1b856 --- /dev/null +++ b/data/sample.csv @@ -0,0 +1,11 @@ +Name,Score,Value,Category +Alice,85,120,A +Bob,92,150,B +Charlie,78,95,A +Diana,88,110,C +Eve,95,180,B +Frank,72,85,A +Grace,91,160,C +Henry,83,105,B +Iris,89,140,A +Jack,76,90,C diff --git a/my_dh_cli/README.md b/my_dh_cli/README.md new file mode 100644 index 0000000..8e79176 --- /dev/null +++ b/my_dh_cli/README.md @@ -0,0 +1,57 @@ +# My Deephaven CLI + +A CLI-only package providing command-line tools for data processing with Deephaven. This package is designed to be installed and run as a command-line tool. + +## Installation + +```shell +pip install . +``` + +Or in editable mode for development: + +```shell +pip install -e . +``` + +## Usage + +> [!NOTE] +> CLI functions require a Deephaven server running in the same Python process. Use the functions within a Python session where the server is already started. + +```shell +pip install -e . +python +``` + +Then in Python: + +```python +# Start the Deephaven server +from deephaven_server import Server +server = Server(port=10000, jvm_args=["-Xmx4g"]) +server.start() + +# Now use the CLI function +from my_dh_cli.cli import my_dh_query +result = my_dh_query("../data/sample.csv", verbose=True) +print(f"Processed {result.size} rows") +``` + +## Commands + +### my-dh-query + +Process a CSV file with Deephaven. + +**Arguments:** +- `input_file` - Path to the CSV file to process + +**Options:** +- `--verbose, -v` - Enable verbose output + +## Requirements + +- Python 3.8 or later +- Deephaven Server 0.35.0 or later +- Click 8.0.0 or later diff --git a/my_dh_cli/pyproject.toml b/my_dh_cli/pyproject.toml new file mode 100644 index 0000000..32c7f84 --- /dev/null +++ b/my_dh_cli/pyproject.toml @@ -0,0 +1,20 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "my_dh_cli" +version = "0.1.0" +description = "Command-line tool for data processing" +readme = "README.md" +requires-python = ">=3.8" +dependencies = [ + "deephaven-server>=0.35.0", + "click>=8.0.0", +] + +[project.scripts] +my-dh-query = "my_dh_cli.cli:app" + +[tool.setuptools.packages.find] +where = ["src"] diff --git a/my_dh_cli/src/my_dh_cli.egg-info/PKG-INFO b/my_dh_cli/src/my_dh_cli.egg-info/PKG-INFO new file mode 100644 index 0000000..88f54ca --- /dev/null +++ b/my_dh_cli/src/my_dh_cli.egg-info/PKG-INFO @@ -0,0 +1,59 @@ +Metadata-Version: 2.4 +Name: my_dh_cli +Version: 0.1.0 +Summary: Command-line tool for data processing +Requires-Python: >=3.8 +Description-Content-Type: text/markdown +Requires-Dist: deephaven-server>=0.35.0 +Requires-Dist: click>=8.0.0 + +# My Deephaven CLI + +A CLI-only package providing command-line tools for data processing with Deephaven. This package is designed to be installed and run as a command-line tool. + +## Installation + +```shell +pip install . +``` + +Or in editable mode for development: + +```shell +pip install -e . +``` + +## Usage + +After installation, use the command-line tool: + +```shell +my-dh-query input_data.csv +my-dh-query input_data.csv --verbose +``` + +### Module Execution + +You can also run without installation using module execution: + +```shell +python -m my_dh_cli input_data.csv +``` + +## Commands + +### my-dh-query + +Process a CSV file with Deephaven. + +**Arguments:** +- `input_file` - Path to the CSV file to process + +**Options:** +- `--verbose, -v` - Enable verbose output + +## Requirements + +- Python 3.8 or later +- Deephaven Server 0.35.0 or later +- Click 8.0.0 or later diff --git a/my_dh_cli/src/my_dh_cli.egg-info/SOURCES.txt b/my_dh_cli/src/my_dh_cli.egg-info/SOURCES.txt new file mode 100644 index 0000000..03fe790 --- /dev/null +++ b/my_dh_cli/src/my_dh_cli.egg-info/SOURCES.txt @@ -0,0 +1,11 @@ +README.md +pyproject.toml +src/my_dh_cli/__init__.py +src/my_dh_cli/__main__.py +src/my_dh_cli/cli.py +src/my_dh_cli.egg-info/PKG-INFO +src/my_dh_cli.egg-info/SOURCES.txt +src/my_dh_cli.egg-info/dependency_links.txt +src/my_dh_cli.egg-info/entry_points.txt +src/my_dh_cli.egg-info/requires.txt +src/my_dh_cli.egg-info/top_level.txt \ No newline at end of file diff --git a/my_dh_cli/src/my_dh_cli.egg-info/dependency_links.txt b/my_dh_cli/src/my_dh_cli.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/my_dh_cli/src/my_dh_cli.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/my_dh_cli/src/my_dh_cli.egg-info/entry_points.txt b/my_dh_cli/src/my_dh_cli.egg-info/entry_points.txt new file mode 100644 index 0000000..d94621b --- /dev/null +++ b/my_dh_cli/src/my_dh_cli.egg-info/entry_points.txt @@ -0,0 +1,2 @@ +[console_scripts] +my-dh-query = my_dh_cli.cli:app diff --git a/my_dh_cli/src/my_dh_cli.egg-info/requires.txt b/my_dh_cli/src/my_dh_cli.egg-info/requires.txt new file mode 100644 index 0000000..18973ba --- /dev/null +++ b/my_dh_cli/src/my_dh_cli.egg-info/requires.txt @@ -0,0 +1,2 @@ +deephaven-server>=0.35.0 +click>=8.0.0 diff --git a/my_dh_cli/src/my_dh_cli.egg-info/top_level.txt b/my_dh_cli/src/my_dh_cli.egg-info/top_level.txt new file mode 100644 index 0000000..5ee37e7 --- /dev/null +++ b/my_dh_cli/src/my_dh_cli.egg-info/top_level.txt @@ -0,0 +1 @@ +my_dh_cli diff --git a/my_dh_cli/src/my_dh_cli/__init__.py b/my_dh_cli/src/my_dh_cli/__init__.py new file mode 100644 index 0000000..cd9785f --- /dev/null +++ b/my_dh_cli/src/my_dh_cli/__init__.py @@ -0,0 +1,3 @@ +"""My Deephaven package for data processing.""" + +__version__ = "0.1.0" diff --git a/my_dh_cli/src/my_dh_cli/__main__.py b/my_dh_cli/src/my_dh_cli/__main__.py new file mode 100644 index 0000000..ff7364e --- /dev/null +++ b/my_dh_cli/src/my_dh_cli/__main__.py @@ -0,0 +1,4 @@ +from my_dh_cli.cli import app + +if __name__ == "__main__": + app() diff --git a/my_dh_cli/src/my_dh_cli/__pycache__/__init__.cpython-312.pyc b/my_dh_cli/src/my_dh_cli/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..29c5838225023eddf3bbe04c328e8a21fd7c23bb GIT binary patch literal 267 zcmX@j%ge<81a+?qGTnjnV-N=h7@>^MJV3^Dh7^V_$Jha<^>WT Vm>C%vZ!+jV;1g}+E@B0W008cWPgMW_ literal 0 HcmV?d00001 diff --git a/my_dh_cli/src/my_dh_cli/__pycache__/cli.cpython-312.pyc b/my_dh_cli/src/my_dh_cli/__pycache__/cli.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8d23a93f7e28e8e181d948dc9fd9185805e89af3 GIT binary patch literal 1606 zcmah}&2QX96rZuZUhh}ZkVcRwDQ=ol7HKz7!X=afN|QrrtBMqm0+Ho-W_P{u+GfU1 zS!*M0L{ZaIb8DmuX?to=a^T8;fW&1*f>cZotq=!JECg}ri5c&jO{+?btoPo$H}C!4 zy!XaG7VIAg={muhk3Dc(S`kq!YEgUA-Fj;XZHhO7A zkO@Xv0m0$-jId%sBv_yZ*98a~kc@iV9<8bzOMQ;S2Dl55^tBjY8Ug_{ga(NB>taK! z#trE^0+cAjeyP?ahIQ(Hu#yd_`kZDkeB~&9Ls|s+RPg?XSVyX^&Mkhd^4qJrjZ_zt zl0#}tP0gCil%M;U)K!P+b|Xt4R0sBH?Aq!%!hCl4j7gAaj*C2By@s%W5DN zi^br@3y!~H;YGtCSbazB^4HX}+>ImomQI6o&kZ9rNT|fQPV;h@LKwRheGS`$7dA{7 zmT9bp5;iJMDElr;0f!Pb1DsC~-XOw9q zs$Oxi=hdhG11h+6uj1GRS2tGmGP}fdt*)Vp#*n8eF^=q*rs&qgOxGS2i5z1fJO%n0 zBTev|H2F9=@KDL!PTxvzecBppWq0P=DGZ#>S8Y*qJjCz}h8a>KWmZ!Nb^pKnjS zyScodd#|~0#A~S~-_P93Y_IRkw};OjC~rQ>3^gwwviRT{tDua(FsVnGBzbig2dP4# zH^G5WSPMqxZJpI#?dep<_t>cCg)nJStz_xtP^w_d0Vm~$Y{36lRFEHT_@On=c zfCiiT_}Q)t9zPSUD))G1AsA0Ndvrox490tnj^;H$CoBuQ@-?HYCLB>N-cU|%llSa5{f{`ZpI}HX#x5rwa?B@#2g@;n= z=EaSR?ZMMONw1J0?xbj1O*d^#%fp3yI>HJ?>XA@daU3g3AWX8HYV47OicZRY4clHQ zUDdrxC_65H8xbgvWa~AoY2Sb-bd=N+;CM$*>A5MoSHp=Z$E%JW*XlZ z-(20E**V^x`QQLv>Ifn{v&G^u)|!gg&dgtYcPIkkGUW6eCqo^XDLYvpWjBK##nGRo O None: + """Process data with Deephaven.""" + result = my_dh_query(input_file, verbose) + click.echo("Processing complete!") + + +if __name__ == "__main__": + app() diff --git a/my_dh_library/README.md b/my_dh_library/README.md new file mode 100644 index 0000000..d275e18 --- /dev/null +++ b/my_dh_library/README.md @@ -0,0 +1,55 @@ +# My Deephaven Library + +A library-only package providing reusable Deephaven query functions. This package contains no CLI tools - it's designed to be imported and used as a library in other Python projects. + +## Installation + +```shell +pip install . +``` + +Or in editable mode for development: + +```shell +pip install -e . +``` + +## Usage + +> [!NOTE] +> All Deephaven functionality requires a running server. Start the server before importing Deephaven modules. + +Import and use the library functions in your Python code: + +```python +# Start the Deephaven server +from deephaven_server import Server +server = Server(port=10000, jvm_args=["-Xmx4g"]) +server.start() + +# Now use the library functions +from my_dh_library.queries import filter_by_threshold, add_computed_columns +from deephaven import read_csv + +data = read_csv("data.csv") +filtered = filter_by_threshold(data, "Score", 75.0) +enhanced = add_computed_columns(filtered) +``` + +## Available Functions + +### Query Functions (`my_dh_library.queries`) + +- `filter_by_threshold(table, column, threshold)` - Filter table rows where column value exceeds threshold +- `add_computed_columns(table)` - Add commonly used computed columns to a table +- `summarize_by_group(table, group_col, value_col)` - Create summary statistics grouped by a column + +### Utility Functions (`my_dh_library.utils`) + +- `validate_columns(table, required_columns)` - Check if table has all required columns +- `get_table_info(table)` - Get basic information about a table + +## Requirements + +- Python 3.8 or later +- Deephaven Server 0.35.0 or later diff --git a/my_dh_library/pyproject.toml b/my_dh_library/pyproject.toml new file mode 100644 index 0000000..3fa6b90 --- /dev/null +++ b/my_dh_library/pyproject.toml @@ -0,0 +1,16 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "my_dh_library" +version = "0.1.0" +description = "Reusable Deephaven query functions" +readme = "README.md" +requires-python = ">=3.8" +dependencies = [ + "deephaven-server>=0.35.0", +] + +[tool.setuptools.packages.find] +where = ["src"] diff --git a/my_dh_library/src/my_dh_library.egg-info/PKG-INFO b/my_dh_library/src/my_dh_library.egg-info/PKG-INFO new file mode 100644 index 0000000..959f753 --- /dev/null +++ b/my_dh_library/src/my_dh_library.egg-info/PKG-INFO @@ -0,0 +1,55 @@ +Metadata-Version: 2.4 +Name: my_dh_library +Version: 0.1.0 +Summary: Reusable Deephaven query functions +Requires-Python: >=3.8 +Description-Content-Type: text/markdown +Requires-Dist: deephaven-server>=0.35.0 + +# My Deephaven Library + +A library-only package providing reusable Deephaven query functions. This package contains no CLI tools - it's designed to be imported and used as a library in other Python projects. + +## Installation + +```shell +pip install . +``` + +Or in editable mode for development: + +```shell +pip install -e . +``` + +## Usage + +Import and use the library functions in your Python code: + +```python +from my_dh_library.queries import filter_by_threshold, add_computed_columns +from deephaven import read_csv + +# Use the library functions +data = read_csv("data.csv") +filtered = filter_by_threshold(data, "Score", 75.0) +enhanced = add_computed_columns(filtered) +``` + +## Available Functions + +### Query Functions (`my_dh_package.queries`) + +- `filter_by_threshold(table, column, threshold)` - Filter table rows where column value exceeds threshold +- `add_computed_columns(table)` - Add commonly used computed columns to a table +- `summarize_by_group(table, group_col, value_col)` - Create summary statistics grouped by a column + +### Utility Functions (`my_dh_package.utils`) + +- `validate_columns(table, required_columns)` - Check if table has all required columns +- `get_table_info(table)` - Get basic information about a table + +## Requirements + +- Python 3.8 or later +- Deephaven Server 0.35.0 or later diff --git a/my_dh_library/src/my_dh_library.egg-info/SOURCES.txt b/my_dh_library/src/my_dh_library.egg-info/SOURCES.txt new file mode 100644 index 0000000..29745ee --- /dev/null +++ b/my_dh_library/src/my_dh_library.egg-info/SOURCES.txt @@ -0,0 +1,10 @@ +README.md +pyproject.toml +src/my_dh_library/__init__.py +src/my_dh_library/queries.py +src/my_dh_library/utils.py +src/my_dh_library.egg-info/PKG-INFO +src/my_dh_library.egg-info/SOURCES.txt +src/my_dh_library.egg-info/dependency_links.txt +src/my_dh_library.egg-info/requires.txt +src/my_dh_library.egg-info/top_level.txt \ No newline at end of file diff --git a/my_dh_library/src/my_dh_library.egg-info/dependency_links.txt b/my_dh_library/src/my_dh_library.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/my_dh_library/src/my_dh_library.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/my_dh_library/src/my_dh_library.egg-info/requires.txt b/my_dh_library/src/my_dh_library.egg-info/requires.txt new file mode 100644 index 0000000..b68ceb0 --- /dev/null +++ b/my_dh_library/src/my_dh_library.egg-info/requires.txt @@ -0,0 +1 @@ +deephaven-server>=0.35.0 diff --git a/my_dh_library/src/my_dh_library.egg-info/top_level.txt b/my_dh_library/src/my_dh_library.egg-info/top_level.txt new file mode 100644 index 0000000..e663cac --- /dev/null +++ b/my_dh_library/src/my_dh_library.egg-info/top_level.txt @@ -0,0 +1 @@ +my_dh_library diff --git a/my_dh_library/src/my_dh_library/__init__.py b/my_dh_library/src/my_dh_library/__init__.py new file mode 100644 index 0000000..6e191c0 --- /dev/null +++ b/my_dh_library/src/my_dh_library/__init__.py @@ -0,0 +1,7 @@ +"""My Deephaven package for data processing.""" + +__version__ = "0.1.0" + +from my_dh_library.queries import filter_by_threshold, add_computed_columns, summarize_by_group + +__all__ = ["filter_by_threshold", "add_computed_columns", "summarize_by_group"] diff --git a/my_dh_library/src/my_dh_library/__pycache__/__init__.cpython-312.pyc b/my_dh_library/src/my_dh_library/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b37569a3e1e4e05ff48dffac6d0a09febdba9acd GIT binary patch literal 427 zcmZ8cu};G<6m*(|qE=-9DlujxLzA*2CRSi#Aj^{*o5o9bn5Bir literal 0 HcmV?d00001 diff --git a/my_dh_library/src/my_dh_library/__pycache__/queries.cpython-312.pyc b/my_dh_library/src/my_dh_library/__pycache__/queries.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ed1febe3b02bfeabb273101e9b1942c20bb341de GIT binary patch literal 1355 zcmZ{jJ&fE$6vt=mwfCC~u2btj@7BzYflQMA0sym}()8v|SA&sxytYm`)AWq9)V!I<$SKwQHzz z8MkN$VpGLk$m_wU$?L;;yBJu*^8GJZF5Gd*$W_MDiM!8Y@?Fk&LH6?4%Yq~p_Fqzm zg)8H_OstGd8PiL`Jc_GsHsMT6l8{y=XIai;4<52y26!@_+pyh72k0hV-YuO2`~aSI zpglkj^~cBe*N~-^AAKBz8RJBiBs{q#$gK(Ej5Ni`z8mI@uZz?Bwk0 z*=L8RH-7DJ9=0~;aF4tfNy!H&j_csOilZD4UweU|UfP!_1zJ&*#9={l!BmvynQ~+Y zfm=e{2C-e9zMACFJV2)ID{`VqFu-sOECLU-R^FITQ4d7MGyNxNhP>RiZwTsOuWP$lvSLSJBo*DnhL#5zfRJUG( literal 0 HcmV?d00001 diff --git a/my_dh_library/src/my_dh_library/queries.py b/my_dh_library/src/my_dh_library/queries.py new file mode 100644 index 0000000..605439e --- /dev/null +++ b/my_dh_library/src/my_dh_library/queries.py @@ -0,0 +1,30 @@ +"""Reusable Deephaven query functions.""" + +from deephaven.table import Table + + +def filter_by_threshold(table: Table, column: str, threshold: float) -> Table: + """Filter table rows where column value exceeds threshold.""" + return table.where(f"{column} > {threshold}") + + +def add_computed_columns(table: Table) -> Table: + """Add commonly used computed columns to a table.""" + return table.update( + [ + "DoubleValue = Value * 2", + "IsHigh = Value > 100", + ] + ) + + +def summarize_by_group(table: Table, group_col: str, value_col: str) -> Table: + """Create summary statistics grouped by a column.""" + return table.agg_by( + [ + f"Sum = sum({value_col})", + f"Avg = avg({value_col})", + f"Count = count()", + ], + by=[group_col], + ) diff --git a/my_dh_library/src/my_dh_library/utils.py b/my_dh_library/src/my_dh_library/utils.py new file mode 100644 index 0000000..b8d2a88 --- /dev/null +++ b/my_dh_library/src/my_dh_library/utils.py @@ -0,0 +1,18 @@ +"""Utility functions for working with Deephaven tables.""" + +from deephaven.table import Table + + +def validate_columns(table: Table, required_columns: list[str]) -> bool: + """Check if table has all required columns.""" + table_columns = [col.name for col in table.columns] + return all(col in table_columns for col in required_columns) + + +def get_table_info(table: Table) -> dict: + """Get basic information about a table.""" + return { + "num_rows": table.size, + "num_columns": len(table.columns), + "columns": [col.name for col in table.columns], + } diff --git a/my_dh_toolkit/README.md b/my_dh_toolkit/README.md new file mode 100644 index 0000000..ee3f6ea --- /dev/null +++ b/my_dh_toolkit/README.md @@ -0,0 +1,104 @@ +# My Deephaven Toolkit + +A combined package providing both reusable library code and command-line functions for Deephaven. This package can be used as both a library (imported in Python code) and as CLI functions (called within a Python session). + +## Installation + +```shell +pip install . +``` + +Or in editable mode for development: + +```shell +pip install -e . +``` + +## Usage as a Library + +> [!NOTE] +> All Deephaven functionality requires a running server. Start the server before importing Deephaven modules. + +Import and use the library functions in your Python code: + +```python +# Start the Deephaven server +from deephaven_server import Server +server = Server(port=10000, jvm_args=["-Xmx4g"]) +server.start() + +# Now use the library functions +from my_dh_toolkit.queries import filter_by_threshold, add_computed_columns +from my_dh_toolkit import my_dh_query, batch_process +from deephaven import read_csv + +data = read_csv("data.csv") +filtered = filter_by_threshold(data, "Score", 75.0) + +# Or use the exported functions +result = my_dh_query("data.csv", verbose=True) +``` + +## Usage as CLI Functions + +> [!NOTE] +> CLI functions require a Deephaven server running in the same Python process. Use them within a Python session where the server is already started. + +```python +# Start the Deephaven server +from deephaven_server import Server +server = Server(port=10000, jvm_args=["-Xmx4g"]) +server.start() + +# Use the CLI functions +from my_dh_toolkit import my_dh_query, batch_process +result = my_dh_query("../data/sample.csv", verbose=True) +batch_process("../data/batch/", "./output", verbose=True) +``` + +## Commands + +### my-dh-query + +Process a single CSV file with Deephaven. + +**Arguments:** +- `input_file` - Path to the CSV file to process + +**Options:** +- `--verbose, -v` - Enable verbose output + +### my-dh-process + +Batch process multiple CSV files from a directory. + +**Arguments:** +- `directory` - Directory containing CSV files to process + +**Options:** +- `--output, -o` - Output directory (default: ./output) +- `--verbose, -v` - Enable verbose output + +## Available Functions + +### Query Functions (`my_dh_toolkit.queries`) + +- `filter_by_threshold(table, column, threshold)` - Filter table rows where column value exceeds threshold +- `add_computed_columns(table)` - Add commonly used computed columns to a table +- `summarize_by_group(table, group_col, value_col)` - Create summary statistics grouped by a column + +### Utility Functions (`my_dh_toolkit.utils`) + +- `validate_columns(table, required_columns)` - Check if table has all required columns +- `get_table_info(table)` - Get basic information about a table + +### Exported Functions (`my_dh_toolkit`) + +- `my_dh_query(input_file, verbose)` - Read and process a CSV file +- `batch_process(directory, output_dir, verbose)` - Process multiple CSV files + +## Requirements + +- Python 3.8 or later +- Deephaven Server 0.35.0 or later +- Click 8.0.0 or later diff --git a/my_dh_toolkit/pyproject.toml b/my_dh_toolkit/pyproject.toml new file mode 100644 index 0000000..81bc624 --- /dev/null +++ b/my_dh_toolkit/pyproject.toml @@ -0,0 +1,21 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "my_dh_toolkit" +version = "0.1.0" +description = "Deephaven library and CLI tools" +readme = "README.md" +requires-python = ">=3.8" +dependencies = [ + "deephaven-server>=0.35.0", + "click>=8.0.0", +] + +[project.scripts] +my-dh-query = "my_dh_toolkit.cli:app" +my-dh-process = "my_dh_toolkit.processor:process" + +[tool.setuptools.packages.find] +where = ["src"] diff --git a/my_dh_toolkit/src/my_dh_toolkit/__init__.py b/my_dh_toolkit/src/my_dh_toolkit/__init__.py new file mode 100644 index 0000000..31bf609 --- /dev/null +++ b/my_dh_toolkit/src/my_dh_toolkit/__init__.py @@ -0,0 +1,8 @@ +"""My Deephaven package for data processing.""" + +__version__ = "0.1.0" + +from my_dh_toolkit.cli import my_dh_query +from my_dh_toolkit.processor import batch_process + +__all__ = ["my_dh_query", "batch_process"] diff --git a/my_dh_toolkit/src/my_dh_toolkit/__main__.py b/my_dh_toolkit/src/my_dh_toolkit/__main__.py new file mode 100644 index 0000000..c7407a9 --- /dev/null +++ b/my_dh_toolkit/src/my_dh_toolkit/__main__.py @@ -0,0 +1,4 @@ +from my_dh_toolkit.cli import app + +if __name__ == "__main__": + app() diff --git a/my_dh_toolkit/src/my_dh_toolkit/cli.py b/my_dh_toolkit/src/my_dh_toolkit/cli.py new file mode 100644 index 0000000..7ffd645 --- /dev/null +++ b/my_dh_toolkit/src/my_dh_toolkit/cli.py @@ -0,0 +1,31 @@ +import click + + +def my_dh_query(input_file: str, verbose: bool = False): + """Read a CSV file and perform a simple query operation on the data.""" + from deephaven import read_csv + + if verbose: + click.echo(f"Processing {input_file}...") + + source = read_csv(input_file) + + result = source.update(formulas=["DoubleScore = Score * 2"]) + + if verbose: + click.echo(f"Processed {result.size} rows") + + return result + + +@click.command() +@click.argument("input_file", type=click.Path(exists=True)) +@click.option("--verbose", "-v", is_flag=True, help="Enable verbose output") +def app(input_file: str, verbose: bool) -> None: + """Process data with Deephaven.""" + result = my_dh_query(input_file, verbose) + click.echo("Processing complete!") + + +if __name__ == "__main__": + app() diff --git a/my_dh_toolkit/src/my_dh_toolkit/processor.py b/my_dh_toolkit/src/my_dh_toolkit/processor.py new file mode 100644 index 0000000..009b099 --- /dev/null +++ b/my_dh_toolkit/src/my_dh_toolkit/processor.py @@ -0,0 +1,40 @@ +import click +from pathlib import Path + + +def batch_process(directory: str, output_dir: str, verbose: bool = False) -> None: + """Process multiple CSV files from a directory.""" + from deephaven import read_csv + + input_path = Path(directory) + output_path = Path(output_dir) + output_path.mkdir(exist_ok=True) + + csv_files = list(input_path.glob("*.csv")) + + if verbose: + click.echo(f"Found {len(csv_files)} CSV files to process") + + for csv_file in csv_files: + if verbose: + click.echo(f"Processing {csv_file.name}...") + + table = read_csv(str(csv_file)) + processed = table.update(formulas=["ProcessedScore = Score * 2"]) + + if verbose: + click.echo(f" Processed {processed.size} rows") + + +@click.command() +@click.argument("directory", type=click.Path(exists=True, file_okay=False)) +@click.option("--output", "-o", default="./output", help="Output directory") +@click.option("--verbose", "-v", is_flag=True, help="Enable verbose output") +def process(directory: str, output: str, verbose: bool) -> None: + """Batch process CSV files with Deephaven.""" + batch_process(directory, output, verbose) + click.echo("Batch processing complete!") + + +if __name__ == "__main__": + process() diff --git a/my_dh_toolkit/src/my_dh_toolkit/queries.py b/my_dh_toolkit/src/my_dh_toolkit/queries.py new file mode 100644 index 0000000..605439e --- /dev/null +++ b/my_dh_toolkit/src/my_dh_toolkit/queries.py @@ -0,0 +1,30 @@ +"""Reusable Deephaven query functions.""" + +from deephaven.table import Table + + +def filter_by_threshold(table: Table, column: str, threshold: float) -> Table: + """Filter table rows where column value exceeds threshold.""" + return table.where(f"{column} > {threshold}") + + +def add_computed_columns(table: Table) -> Table: + """Add commonly used computed columns to a table.""" + return table.update( + [ + "DoubleValue = Value * 2", + "IsHigh = Value > 100", + ] + ) + + +def summarize_by_group(table: Table, group_col: str, value_col: str) -> Table: + """Create summary statistics grouped by a column.""" + return table.agg_by( + [ + f"Sum = sum({value_col})", + f"Avg = avg({value_col})", + f"Count = count()", + ], + by=[group_col], + ) diff --git a/my_dh_toolkit/src/my_dh_toolkit/utils.py b/my_dh_toolkit/src/my_dh_toolkit/utils.py new file mode 100644 index 0000000..b8d2a88 --- /dev/null +++ b/my_dh_toolkit/src/my_dh_toolkit/utils.py @@ -0,0 +1,18 @@ +"""Utility functions for working with Deephaven tables.""" + +from deephaven.table import Table + + +def validate_columns(table: Table, required_columns: list[str]) -> bool: + """Check if table has all required columns.""" + table_columns = [col.name for col in table.columns] + return all(col in table_columns for col in required_columns) + + +def get_table_info(table: Table) -> dict: + """Get basic information about a table.""" + return { + "num_rows": table.size, + "num_columns": len(table.columns), + "columns": [col.name for col in table.columns], + } From 2059ecb4379d3cb21421f747908a2bc340cf4123 Mon Sep 17 00:00:00 2001 From: elijahpetty Date: Fri, 6 Feb 2026 13:19:09 -0600 Subject: [PATCH 02/13] . --- .gitignore | 2 ++ my_dh_cli/src/my_dh_cli/cli.py | 19 ++++++++++++- my_dh_library/src/my_dh_library/queries.py | 4 +++ my_dh_library/src/my_dh_library/utils.py | 27 ++++++++++++++++--- my_dh_toolkit/src/my_dh_toolkit/cli.py | 19 ++++++++++++- my_dh_toolkit/src/my_dh_toolkit/processor.py | 28 ++++++++++++++++++-- my_dh_toolkit/src/my_dh_toolkit/queries.py | 4 +++ my_dh_toolkit/src/my_dh_toolkit/utils.py | 27 ++++++++++++++++--- 8 files changed, 120 insertions(+), 10 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ef037ef --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.egg-info/ +__pycache__/ \ No newline at end of file diff --git a/my_dh_cli/src/my_dh_cli/cli.py b/my_dh_cli/src/my_dh_cli/cli.py index 7ffd645..d074cfd 100644 --- a/my_dh_cli/src/my_dh_cli/cli.py +++ b/my_dh_cli/src/my_dh_cli/cli.py @@ -4,11 +4,28 @@ def my_dh_query(input_file: str, verbose: bool = False): """Read a CSV file and perform a simple query operation on the data.""" from deephaven import read_csv + from pathlib import Path + + input_path = Path(input_file) + + if not input_path.exists(): + raise click.ClickException(f"Input file does not exist: '{input_path}'") + if not input_path.is_file(): + raise click.ClickException(f"Input path is not a file: '{input_path}'") if verbose: click.echo(f"Processing {input_file}...") - source = read_csv(input_file) + try: + source = read_csv(input_file) + except Exception as e: + raise click.ClickException(f"Failed to read CSV file '{input_file}': {e}") + + if "Score" not in source.columns: + raise click.ClickException( + f"File '{input_path.name}' is missing required column 'Score'. " + f"Available columns: {', '.join(source.columns)}" + ) result = source.update(formulas=["DoubleScore = Score * 2"]) diff --git a/my_dh_library/src/my_dh_library/queries.py b/my_dh_library/src/my_dh_library/queries.py index 605439e..6471685 100644 --- a/my_dh_library/src/my_dh_library/queries.py +++ b/my_dh_library/src/my_dh_library/queries.py @@ -1,15 +1,18 @@ """Reusable Deephaven query functions.""" from deephaven.table import Table +from .utils import validate_columns def filter_by_threshold(table: Table, column: str, threshold: float) -> Table: """Filter table rows where column value exceeds threshold.""" + validate_columns(table, [column], raise_error=True) return table.where(f"{column} > {threshold}") def add_computed_columns(table: Table) -> Table: """Add commonly used computed columns to a table.""" + validate_columns(table, ["Value"], raise_error=True) return table.update( [ "DoubleValue = Value * 2", @@ -20,6 +23,7 @@ def add_computed_columns(table: Table) -> Table: def summarize_by_group(table: Table, group_col: str, value_col: str) -> Table: """Create summary statistics grouped by a column.""" + validate_columns(table, [group_col, value_col], raise_error=True) return table.agg_by( [ f"Sum = sum({value_col})", diff --git a/my_dh_library/src/my_dh_library/utils.py b/my_dh_library/src/my_dh_library/utils.py index b8d2a88..f38c859 100644 --- a/my_dh_library/src/my_dh_library/utils.py +++ b/my_dh_library/src/my_dh_library/utils.py @@ -3,10 +3,31 @@ from deephaven.table import Table -def validate_columns(table: Table, required_columns: list[str]) -> bool: - """Check if table has all required columns.""" +def validate_columns(table: Table, required_columns: list[str], raise_error: bool = False) -> bool: + """Check if table has all required columns. + + Args: + table: The table to validate + required_columns: List of column names that must be present + raise_error: If True, raises ValueError when columns are missing + + Returns: + True if all columns are present, False otherwise + + Raises: + ValueError: If raise_error is True and columns are missing + """ table_columns = [col.name for col in table.columns] - return all(col in table_columns for col in required_columns) + missing = [col for col in required_columns if col not in table_columns] + + if missing: + if raise_error: + raise ValueError( + f"Column(s) {missing} not found in table. " + f"Available columns: {', '.join(table_columns)}" + ) + return False + return True def get_table_info(table: Table) -> dict: diff --git a/my_dh_toolkit/src/my_dh_toolkit/cli.py b/my_dh_toolkit/src/my_dh_toolkit/cli.py index 7ffd645..d074cfd 100644 --- a/my_dh_toolkit/src/my_dh_toolkit/cli.py +++ b/my_dh_toolkit/src/my_dh_toolkit/cli.py @@ -4,11 +4,28 @@ def my_dh_query(input_file: str, verbose: bool = False): """Read a CSV file and perform a simple query operation on the data.""" from deephaven import read_csv + from pathlib import Path + + input_path = Path(input_file) + + if not input_path.exists(): + raise click.ClickException(f"Input file does not exist: '{input_path}'") + if not input_path.is_file(): + raise click.ClickException(f"Input path is not a file: '{input_path}'") if verbose: click.echo(f"Processing {input_file}...") - source = read_csv(input_file) + try: + source = read_csv(input_file) + except Exception as e: + raise click.ClickException(f"Failed to read CSV file '{input_file}': {e}") + + if "Score" not in source.columns: + raise click.ClickException( + f"File '{input_path.name}' is missing required column 'Score'. " + f"Available columns: {', '.join(source.columns)}" + ) result = source.update(formulas=["DoubleScore = Score * 2"]) diff --git a/my_dh_toolkit/src/my_dh_toolkit/processor.py b/my_dh_toolkit/src/my_dh_toolkit/processor.py index 009b099..1e50cea 100644 --- a/my_dh_toolkit/src/my_dh_toolkit/processor.py +++ b/my_dh_toolkit/src/my_dh_toolkit/processor.py @@ -8,7 +8,18 @@ def batch_process(directory: str, output_dir: str, verbose: bool = False) -> Non input_path = Path(directory) output_path = Path(output_dir) - output_path.mkdir(exist_ok=True) + + if not input_path.exists(): + raise click.ClickException(f"Input directory does not exist: '{input_path}'") + if not input_path.is_dir(): + raise click.ClickException(f"Input path is not a directory: '{input_path}'") + + try: + output_path.mkdir(parents=True, exist_ok=True) + except PermissionError: + raise click.ClickException(f"Permission denied: Cannot create output directory '{output_path}'") + except OSError as e: + raise click.ClickException(f"Failed to create output directory '{output_path}': {e}") csv_files = list(input_path.glob("*.csv")) @@ -20,10 +31,23 @@ def batch_process(directory: str, output_dir: str, verbose: bool = False) -> Non click.echo(f"Processing {csv_file.name}...") table = read_csv(str(csv_file)) + + if "Score" not in table.columns: + raise click.ClickException( + f"File '{csv_file.name}' is missing required column 'Score'. " + f"Available columns: {', '.join(table.columns)}" + ) + processed = table.update(formulas=["ProcessedScore = Score * 2"]) + + output_file = output_path / f"processed_{csv_file.name}" + try: + processed.to_csv(str(output_file)) + except Exception as e: + raise click.ClickException(f"Failed to write output file '{output_file}': {e}") if verbose: - click.echo(f" Processed {processed.size} rows") + click.echo(f" Processed {processed.size} rows -> {output_file.name}") @click.command() diff --git a/my_dh_toolkit/src/my_dh_toolkit/queries.py b/my_dh_toolkit/src/my_dh_toolkit/queries.py index 605439e..6471685 100644 --- a/my_dh_toolkit/src/my_dh_toolkit/queries.py +++ b/my_dh_toolkit/src/my_dh_toolkit/queries.py @@ -1,15 +1,18 @@ """Reusable Deephaven query functions.""" from deephaven.table import Table +from .utils import validate_columns def filter_by_threshold(table: Table, column: str, threshold: float) -> Table: """Filter table rows where column value exceeds threshold.""" + validate_columns(table, [column], raise_error=True) return table.where(f"{column} > {threshold}") def add_computed_columns(table: Table) -> Table: """Add commonly used computed columns to a table.""" + validate_columns(table, ["Value"], raise_error=True) return table.update( [ "DoubleValue = Value * 2", @@ -20,6 +23,7 @@ def add_computed_columns(table: Table) -> Table: def summarize_by_group(table: Table, group_col: str, value_col: str) -> Table: """Create summary statistics grouped by a column.""" + validate_columns(table, [group_col, value_col], raise_error=True) return table.agg_by( [ f"Sum = sum({value_col})", diff --git a/my_dh_toolkit/src/my_dh_toolkit/utils.py b/my_dh_toolkit/src/my_dh_toolkit/utils.py index b8d2a88..f38c859 100644 --- a/my_dh_toolkit/src/my_dh_toolkit/utils.py +++ b/my_dh_toolkit/src/my_dh_toolkit/utils.py @@ -3,10 +3,31 @@ from deephaven.table import Table -def validate_columns(table: Table, required_columns: list[str]) -> bool: - """Check if table has all required columns.""" +def validate_columns(table: Table, required_columns: list[str], raise_error: bool = False) -> bool: + """Check if table has all required columns. + + Args: + table: The table to validate + required_columns: List of column names that must be present + raise_error: If True, raises ValueError when columns are missing + + Returns: + True if all columns are present, False otherwise + + Raises: + ValueError: If raise_error is True and columns are missing + """ table_columns = [col.name for col in table.columns] - return all(col in table_columns for col in required_columns) + missing = [col for col in required_columns if col not in table_columns] + + if missing: + if raise_error: + raise ValueError( + f"Column(s) {missing} not found in table. " + f"Available columns: {', '.join(table_columns)}" + ) + return False + return True def get_table_info(table: Table) -> dict: From 1bfc5d312c17898dba4cbdf3bdf9231a5481a369 Mon Sep 17 00:00:00 2001 From: elijahpetty Date: Fri, 6 Feb 2026 13:40:33 -0600 Subject: [PATCH 03/13] . --- my_dh_cli/README.md | 2 +- my_dh_cli/data/batch/file1.csv | 4 ++ my_dh_cli/data/batch/file2.csv | 4 ++ my_dh_cli/data/batch/file3.csv | 5 +++ my_dh_cli/data/sample.csv | 11 +++++ my_dh_library/README.md | 2 +- .../build/lib/my_dh_library/__init__.py | 7 ++++ .../build/lib/my_dh_library/queries.py | 34 +++++++++++++++ .../build/lib/my_dh_library/utils.py | 39 ++++++++++++++++++ my_dh_library/data/batch/file1.csv | 4 ++ my_dh_library/data/batch/file2.csv | 4 ++ my_dh_library/data/batch/file3.csv | 5 +++ my_dh_library/data/sample.csv | 11 +++++ .../src/my_dh_library.egg-info/PKG-INFO | 16 +++++-- .../__pycache__/queries.cpython-312.pyc | Bin 1355 -> 1594 bytes my_dh_toolkit/README.md | 8 ++-- my_dh_toolkit/data/batch/file1.csv | 4 ++ my_dh_toolkit/data/batch/file2.csv | 4 ++ my_dh_toolkit/data/batch/file3.csv | 5 +++ my_dh_toolkit/data/sample.csv | 11 +++++ 20 files changed, 170 insertions(+), 10 deletions(-) create mode 100644 my_dh_cli/data/batch/file1.csv create mode 100644 my_dh_cli/data/batch/file2.csv create mode 100644 my_dh_cli/data/batch/file3.csv create mode 100644 my_dh_cli/data/sample.csv create mode 100644 my_dh_library/build/lib/my_dh_library/__init__.py create mode 100644 my_dh_library/build/lib/my_dh_library/queries.py create mode 100644 my_dh_library/build/lib/my_dh_library/utils.py create mode 100644 my_dh_library/data/batch/file1.csv create mode 100644 my_dh_library/data/batch/file2.csv create mode 100644 my_dh_library/data/batch/file3.csv create mode 100644 my_dh_library/data/sample.csv create mode 100644 my_dh_toolkit/data/batch/file1.csv create mode 100644 my_dh_toolkit/data/batch/file2.csv create mode 100644 my_dh_toolkit/data/batch/file3.csv create mode 100644 my_dh_toolkit/data/sample.csv diff --git a/my_dh_cli/README.md b/my_dh_cli/README.md index 8e79176..66e3a4a 100644 --- a/my_dh_cli/README.md +++ b/my_dh_cli/README.md @@ -34,7 +34,7 @@ server.start() # Now use the CLI function from my_dh_cli.cli import my_dh_query -result = my_dh_query("../data/sample.csv", verbose=True) +result = my_dh_query("data/sample.csv", verbose=True) print(f"Processed {result.size} rows") ``` diff --git a/my_dh_cli/data/batch/file1.csv b/my_dh_cli/data/batch/file1.csv new file mode 100644 index 0000000..fe9fae6 --- /dev/null +++ b/my_dh_cli/data/batch/file1.csv @@ -0,0 +1,4 @@ +Name,Score,Value,Category +Alice,85,120,A +Bob,92,150,B +Charlie,78,95,A diff --git a/my_dh_cli/data/batch/file2.csv b/my_dh_cli/data/batch/file2.csv new file mode 100644 index 0000000..e270a59 --- /dev/null +++ b/my_dh_cli/data/batch/file2.csv @@ -0,0 +1,4 @@ +Name,Score,Value,Category +Diana,88,110,C +Eve,95,180,B +Frank,72,85,A diff --git a/my_dh_cli/data/batch/file3.csv b/my_dh_cli/data/batch/file3.csv new file mode 100644 index 0000000..692c4a2 --- /dev/null +++ b/my_dh_cli/data/batch/file3.csv @@ -0,0 +1,5 @@ +Name,Score,Value,Category +Grace,91,160,C +Henry,83,105,B +Iris,89,140,A +Jack,76,90,C diff --git a/my_dh_cli/data/sample.csv b/my_dh_cli/data/sample.csv new file mode 100644 index 0000000..1d1b856 --- /dev/null +++ b/my_dh_cli/data/sample.csv @@ -0,0 +1,11 @@ +Name,Score,Value,Category +Alice,85,120,A +Bob,92,150,B +Charlie,78,95,A +Diana,88,110,C +Eve,95,180,B +Frank,72,85,A +Grace,91,160,C +Henry,83,105,B +Iris,89,140,A +Jack,76,90,C diff --git a/my_dh_library/README.md b/my_dh_library/README.md index d275e18..4362a1c 100644 --- a/my_dh_library/README.md +++ b/my_dh_library/README.md @@ -31,7 +31,7 @@ server.start() from my_dh_library.queries import filter_by_threshold, add_computed_columns from deephaven import read_csv -data = read_csv("data.csv") +data = read_csv("data/sample.csv") filtered = filter_by_threshold(data, "Score", 75.0) enhanced = add_computed_columns(filtered) ``` diff --git a/my_dh_library/build/lib/my_dh_library/__init__.py b/my_dh_library/build/lib/my_dh_library/__init__.py new file mode 100644 index 0000000..6e191c0 --- /dev/null +++ b/my_dh_library/build/lib/my_dh_library/__init__.py @@ -0,0 +1,7 @@ +"""My Deephaven package for data processing.""" + +__version__ = "0.1.0" + +from my_dh_library.queries import filter_by_threshold, add_computed_columns, summarize_by_group + +__all__ = ["filter_by_threshold", "add_computed_columns", "summarize_by_group"] diff --git a/my_dh_library/build/lib/my_dh_library/queries.py b/my_dh_library/build/lib/my_dh_library/queries.py new file mode 100644 index 0000000..6471685 --- /dev/null +++ b/my_dh_library/build/lib/my_dh_library/queries.py @@ -0,0 +1,34 @@ +"""Reusable Deephaven query functions.""" + +from deephaven.table import Table +from .utils import validate_columns + + +def filter_by_threshold(table: Table, column: str, threshold: float) -> Table: + """Filter table rows where column value exceeds threshold.""" + validate_columns(table, [column], raise_error=True) + return table.where(f"{column} > {threshold}") + + +def add_computed_columns(table: Table) -> Table: + """Add commonly used computed columns to a table.""" + validate_columns(table, ["Value"], raise_error=True) + return table.update( + [ + "DoubleValue = Value * 2", + "IsHigh = Value > 100", + ] + ) + + +def summarize_by_group(table: Table, group_col: str, value_col: str) -> Table: + """Create summary statistics grouped by a column.""" + validate_columns(table, [group_col, value_col], raise_error=True) + return table.agg_by( + [ + f"Sum = sum({value_col})", + f"Avg = avg({value_col})", + f"Count = count()", + ], + by=[group_col], + ) diff --git a/my_dh_library/build/lib/my_dh_library/utils.py b/my_dh_library/build/lib/my_dh_library/utils.py new file mode 100644 index 0000000..f38c859 --- /dev/null +++ b/my_dh_library/build/lib/my_dh_library/utils.py @@ -0,0 +1,39 @@ +"""Utility functions for working with Deephaven tables.""" + +from deephaven.table import Table + + +def validate_columns(table: Table, required_columns: list[str], raise_error: bool = False) -> bool: + """Check if table has all required columns. + + Args: + table: The table to validate + required_columns: List of column names that must be present + raise_error: If True, raises ValueError when columns are missing + + Returns: + True if all columns are present, False otherwise + + Raises: + ValueError: If raise_error is True and columns are missing + """ + table_columns = [col.name for col in table.columns] + missing = [col for col in required_columns if col not in table_columns] + + if missing: + if raise_error: + raise ValueError( + f"Column(s) {missing} not found in table. " + f"Available columns: {', '.join(table_columns)}" + ) + return False + return True + + +def get_table_info(table: Table) -> dict: + """Get basic information about a table.""" + return { + "num_rows": table.size, + "num_columns": len(table.columns), + "columns": [col.name for col in table.columns], + } diff --git a/my_dh_library/data/batch/file1.csv b/my_dh_library/data/batch/file1.csv new file mode 100644 index 0000000..fe9fae6 --- /dev/null +++ b/my_dh_library/data/batch/file1.csv @@ -0,0 +1,4 @@ +Name,Score,Value,Category +Alice,85,120,A +Bob,92,150,B +Charlie,78,95,A diff --git a/my_dh_library/data/batch/file2.csv b/my_dh_library/data/batch/file2.csv new file mode 100644 index 0000000..e270a59 --- /dev/null +++ b/my_dh_library/data/batch/file2.csv @@ -0,0 +1,4 @@ +Name,Score,Value,Category +Diana,88,110,C +Eve,95,180,B +Frank,72,85,A diff --git a/my_dh_library/data/batch/file3.csv b/my_dh_library/data/batch/file3.csv new file mode 100644 index 0000000..692c4a2 --- /dev/null +++ b/my_dh_library/data/batch/file3.csv @@ -0,0 +1,5 @@ +Name,Score,Value,Category +Grace,91,160,C +Henry,83,105,B +Iris,89,140,A +Jack,76,90,C diff --git a/my_dh_library/data/sample.csv b/my_dh_library/data/sample.csv new file mode 100644 index 0000000..1d1b856 --- /dev/null +++ b/my_dh_library/data/sample.csv @@ -0,0 +1,11 @@ +Name,Score,Value,Category +Alice,85,120,A +Bob,92,150,B +Charlie,78,95,A +Diana,88,110,C +Eve,95,180,B +Frank,72,85,A +Grace,91,160,C +Henry,83,105,B +Iris,89,140,A +Jack,76,90,C diff --git a/my_dh_library/src/my_dh_library.egg-info/PKG-INFO b/my_dh_library/src/my_dh_library.egg-info/PKG-INFO index 959f753..a450448 100644 --- a/my_dh_library/src/my_dh_library.egg-info/PKG-INFO +++ b/my_dh_library/src/my_dh_library.egg-info/PKG-INFO @@ -24,27 +24,35 @@ pip install -e . ## Usage +> [!NOTE] +> All Deephaven functionality requires a running server. Start the server before importing Deephaven modules. + Import and use the library functions in your Python code: ```python +# Start the Deephaven server +from deephaven_server import Server +server = Server(port=10000, jvm_args=["-Xmx4g"]) +server.start() + +# Now use the library functions from my_dh_library.queries import filter_by_threshold, add_computed_columns from deephaven import read_csv -# Use the library functions -data = read_csv("data.csv") +data = read_csv("data/sample.csv") filtered = filter_by_threshold(data, "Score", 75.0) enhanced = add_computed_columns(filtered) ``` ## Available Functions -### Query Functions (`my_dh_package.queries`) +### Query Functions (`my_dh_library.queries`) - `filter_by_threshold(table, column, threshold)` - Filter table rows where column value exceeds threshold - `add_computed_columns(table)` - Add commonly used computed columns to a table - `summarize_by_group(table, group_col, value_col)` - Create summary statistics grouped by a column -### Utility Functions (`my_dh_package.utils`) +### Utility Functions (`my_dh_library.utils`) - `validate_columns(table, required_columns)` - Check if table has all required columns - `get_table_info(table)` - Get basic information about a table diff --git a/my_dh_library/src/my_dh_library/__pycache__/queries.cpython-312.pyc b/my_dh_library/src/my_dh_library/__pycache__/queries.cpython-312.pyc index ed1febe3b02bfeabb273101e9b1942c20bb341de..7bf4d85a749901ec5d4235a21f824fd200f8a80b 100644 GIT binary patch delta 992 zcma)4Pe>GD6#xFt{$D+q8A^?ohv^@tK}P1KKNLhnsnK3^@*B5}nO44;DYOMWMAWI4 zIk+G~M@0#tOLX!Q2wu7f+k()c4&4T}QxNsO*P!0(!h>abRBNz!Pm`xp)W8ia(2+iUmW@#dBk(sziJY%_7 zlvgnmNr>Z7oPxSEJdL^x7wmBkKFRR;oCPN z8lEbNP6?H89t*rd)@dVyQ3*Y4`pu$7C0+))C9;fG=z8<}JIG+c5q$p4*cz^-q*-vS zNlVJ2oTvJzZV<`g(;9#Bz>=0hr3R6VDH9Jo{Pb`exGHpygIPphS;y<~%0%Vb=CNIN zt{VKtC&pC+MW%ig%o5s9TqxpXgZ_)SV1nn#KFV5Pl7UQ(PnfoEjmZ>r4$5R;p-ETs zXVumW9PxG4sJhI8t#z0}z%rd#-?NkxZB=%_s|-BuNieM`?pCgEp50}`)l~bdk(VQ1 z(*4zRU&v`9S*JX$g@|)awyUOzye#^&&@DJc`xd4k_4k-1Tq1e$KMFO}5p0DDu5^k# zEl-ug8?ki)t>J)X@%hu2qy;nTuJ1UeoYP&;^a`$52;S0zT5}~I=}=2S&CKT~@6WX) zJOFKECm2^rqwBDqpoew!c0<&q(zn30HgRt?@u_EUi|w*2)!;I_7*kWHa delta 682 zcmY*VL2DC16rP#4Gf6g^Hnl26imastm!gK&gF>wR0pdk3=2BBYC4Nd<@OS>>mvthC zpf*7VBJ0RzS4GrGq;`-{`zC=)*G4h7I_YH$s8OzN)nC<5f|=uBD(6u0hrk1o6V8PW z%on)fFCy`;Zsgoawg(cdPnUTy^k_`q@dJirTW?AG=%ry;L{l#@*lS8M)(CKbdhQR diff --git a/my_dh_toolkit/README.md b/my_dh_toolkit/README.md index ee3f6ea..bb91728 100644 --- a/my_dh_toolkit/README.md +++ b/my_dh_toolkit/README.md @@ -32,11 +32,11 @@ from my_dh_toolkit.queries import filter_by_threshold, add_computed_columns from my_dh_toolkit import my_dh_query, batch_process from deephaven import read_csv -data = read_csv("data.csv") +data = read_csv("data/sample.csv") filtered = filter_by_threshold(data, "Score", 75.0) # Or use the exported functions -result = my_dh_query("data.csv", verbose=True) +result = my_dh_query("data/sample.csv", verbose=True) ``` ## Usage as CLI Functions @@ -52,8 +52,8 @@ server.start() # Use the CLI functions from my_dh_toolkit import my_dh_query, batch_process -result = my_dh_query("../data/sample.csv", verbose=True) -batch_process("../data/batch/", "./output", verbose=True) +result = my_dh_query("data/sample.csv", verbose=True) +batch_process("data/batch/", "./output", verbose=True) ``` ## Commands diff --git a/my_dh_toolkit/data/batch/file1.csv b/my_dh_toolkit/data/batch/file1.csv new file mode 100644 index 0000000..fe9fae6 --- /dev/null +++ b/my_dh_toolkit/data/batch/file1.csv @@ -0,0 +1,4 @@ +Name,Score,Value,Category +Alice,85,120,A +Bob,92,150,B +Charlie,78,95,A diff --git a/my_dh_toolkit/data/batch/file2.csv b/my_dh_toolkit/data/batch/file2.csv new file mode 100644 index 0000000..e270a59 --- /dev/null +++ b/my_dh_toolkit/data/batch/file2.csv @@ -0,0 +1,4 @@ +Name,Score,Value,Category +Diana,88,110,C +Eve,95,180,B +Frank,72,85,A diff --git a/my_dh_toolkit/data/batch/file3.csv b/my_dh_toolkit/data/batch/file3.csv new file mode 100644 index 0000000..692c4a2 --- /dev/null +++ b/my_dh_toolkit/data/batch/file3.csv @@ -0,0 +1,5 @@ +Name,Score,Value,Category +Grace,91,160,C +Henry,83,105,B +Iris,89,140,A +Jack,76,90,C diff --git a/my_dh_toolkit/data/sample.csv b/my_dh_toolkit/data/sample.csv new file mode 100644 index 0000000..1d1b856 --- /dev/null +++ b/my_dh_toolkit/data/sample.csv @@ -0,0 +1,11 @@ +Name,Score,Value,Category +Alice,85,120,A +Bob,92,150,B +Charlie,78,95,A +Diana,88,110,C +Eve,95,180,B +Frank,72,85,A +Grace,91,160,C +Henry,83,105,B +Iris,89,140,A +Jack,76,90,C From 6da05015ceb9b5288ecc181b11c99e771754a5dc Mon Sep 17 00:00:00 2001 From: elijahpetty Date: Thu, 27 Aug 2026 15:24:12 -0500 Subject: [PATCH 04/13] gitignore file --- .gitignore | 43 +++++++++++- my_dh_cli/src/my_dh_cli.egg-info/PKG-INFO | 59 ---------------- my_dh_cli/src/my_dh_cli.egg-info/SOURCES.txt | 11 --- .../my_dh_cli.egg-info/dependency_links.txt | 1 - .../src/my_dh_cli.egg-info/entry_points.txt | 2 - my_dh_cli/src/my_dh_cli.egg-info/requires.txt | 2 - .../src/my_dh_cli.egg-info/top_level.txt | 1 - .../__pycache__/__init__.cpython-312.pyc | Bin 267 -> 0 bytes .../my_dh_cli/__pycache__/cli.cpython-312.pyc | Bin 1606 -> 0 bytes .../build/lib/my_dh_library/__init__.py | 7 -- .../build/lib/my_dh_library/queries.py | 34 ---------- .../build/lib/my_dh_library/utils.py | 39 ----------- .../src/my_dh_library.egg-info/PKG-INFO | 63 ------------------ .../src/my_dh_library.egg-info/SOURCES.txt | 10 --- .../dependency_links.txt | 1 - .../src/my_dh_library.egg-info/requires.txt | 1 - .../src/my_dh_library.egg-info/top_level.txt | 1 - .../__pycache__/__init__.cpython-312.pyc | Bin 427 -> 0 bytes .../__pycache__/queries.cpython-312.pyc | Bin 1594 -> 0 bytes 19 files changed, 42 insertions(+), 233 deletions(-) delete mode 100644 my_dh_cli/src/my_dh_cli.egg-info/PKG-INFO delete mode 100644 my_dh_cli/src/my_dh_cli.egg-info/SOURCES.txt delete mode 100644 my_dh_cli/src/my_dh_cli.egg-info/dependency_links.txt delete mode 100644 my_dh_cli/src/my_dh_cli.egg-info/entry_points.txt delete mode 100644 my_dh_cli/src/my_dh_cli.egg-info/requires.txt delete mode 100644 my_dh_cli/src/my_dh_cli.egg-info/top_level.txt delete mode 100644 my_dh_cli/src/my_dh_cli/__pycache__/__init__.cpython-312.pyc delete mode 100644 my_dh_cli/src/my_dh_cli/__pycache__/cli.cpython-312.pyc delete mode 100644 my_dh_library/build/lib/my_dh_library/__init__.py delete mode 100644 my_dh_library/build/lib/my_dh_library/queries.py delete mode 100644 my_dh_library/build/lib/my_dh_library/utils.py delete mode 100644 my_dh_library/src/my_dh_library.egg-info/PKG-INFO delete mode 100644 my_dh_library/src/my_dh_library.egg-info/SOURCES.txt delete mode 100644 my_dh_library/src/my_dh_library.egg-info/dependency_links.txt delete mode 100644 my_dh_library/src/my_dh_library.egg-info/requires.txt delete mode 100644 my_dh_library/src/my_dh_library.egg-info/top_level.txt delete mode 100644 my_dh_library/src/my_dh_library/__pycache__/__init__.cpython-312.pyc delete mode 100644 my_dh_library/src/my_dh_library/__pycache__/queries.cpython-312.pyc diff --git a/.gitignore b/.gitignore index ef037ef..f07e50c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,43 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ *.egg-info/ -__pycache__/ \ No newline at end of file +.installed.cfg +*.egg +MANIFEST + +# Virtual environments +venv/ +env/ +ENV/ +.venv + +# IDEs +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Test results +results/ +*.log \ No newline at end of file diff --git a/my_dh_cli/src/my_dh_cli.egg-info/PKG-INFO b/my_dh_cli/src/my_dh_cli.egg-info/PKG-INFO deleted file mode 100644 index 88f54ca..0000000 --- a/my_dh_cli/src/my_dh_cli.egg-info/PKG-INFO +++ /dev/null @@ -1,59 +0,0 @@ -Metadata-Version: 2.4 -Name: my_dh_cli -Version: 0.1.0 -Summary: Command-line tool for data processing -Requires-Python: >=3.8 -Description-Content-Type: text/markdown -Requires-Dist: deephaven-server>=0.35.0 -Requires-Dist: click>=8.0.0 - -# My Deephaven CLI - -A CLI-only package providing command-line tools for data processing with Deephaven. This package is designed to be installed and run as a command-line tool. - -## Installation - -```shell -pip install . -``` - -Or in editable mode for development: - -```shell -pip install -e . -``` - -## Usage - -After installation, use the command-line tool: - -```shell -my-dh-query input_data.csv -my-dh-query input_data.csv --verbose -``` - -### Module Execution - -You can also run without installation using module execution: - -```shell -python -m my_dh_cli input_data.csv -``` - -## Commands - -### my-dh-query - -Process a CSV file with Deephaven. - -**Arguments:** -- `input_file` - Path to the CSV file to process - -**Options:** -- `--verbose, -v` - Enable verbose output - -## Requirements - -- Python 3.8 or later -- Deephaven Server 0.35.0 or later -- Click 8.0.0 or later diff --git a/my_dh_cli/src/my_dh_cli.egg-info/SOURCES.txt b/my_dh_cli/src/my_dh_cli.egg-info/SOURCES.txt deleted file mode 100644 index 03fe790..0000000 --- a/my_dh_cli/src/my_dh_cli.egg-info/SOURCES.txt +++ /dev/null @@ -1,11 +0,0 @@ -README.md -pyproject.toml -src/my_dh_cli/__init__.py -src/my_dh_cli/__main__.py -src/my_dh_cli/cli.py -src/my_dh_cli.egg-info/PKG-INFO -src/my_dh_cli.egg-info/SOURCES.txt -src/my_dh_cli.egg-info/dependency_links.txt -src/my_dh_cli.egg-info/entry_points.txt -src/my_dh_cli.egg-info/requires.txt -src/my_dh_cli.egg-info/top_level.txt \ No newline at end of file diff --git a/my_dh_cli/src/my_dh_cli.egg-info/dependency_links.txt b/my_dh_cli/src/my_dh_cli.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789..0000000 --- a/my_dh_cli/src/my_dh_cli.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/my_dh_cli/src/my_dh_cli.egg-info/entry_points.txt b/my_dh_cli/src/my_dh_cli.egg-info/entry_points.txt deleted file mode 100644 index d94621b..0000000 --- a/my_dh_cli/src/my_dh_cli.egg-info/entry_points.txt +++ /dev/null @@ -1,2 +0,0 @@ -[console_scripts] -my-dh-query = my_dh_cli.cli:app diff --git a/my_dh_cli/src/my_dh_cli.egg-info/requires.txt b/my_dh_cli/src/my_dh_cli.egg-info/requires.txt deleted file mode 100644 index 18973ba..0000000 --- a/my_dh_cli/src/my_dh_cli.egg-info/requires.txt +++ /dev/null @@ -1,2 +0,0 @@ -deephaven-server>=0.35.0 -click>=8.0.0 diff --git a/my_dh_cli/src/my_dh_cli.egg-info/top_level.txt b/my_dh_cli/src/my_dh_cli.egg-info/top_level.txt deleted file mode 100644 index 5ee37e7..0000000 --- a/my_dh_cli/src/my_dh_cli.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -my_dh_cli diff --git a/my_dh_cli/src/my_dh_cli/__pycache__/__init__.cpython-312.pyc b/my_dh_cli/src/my_dh_cli/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 29c5838225023eddf3bbe04c328e8a21fd7c23bb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 267 zcmX@j%ge<81a+?qGTnjnV-N=h7@>^MJV3^Dh7^V_$Jha<^>WT Vm>C%vZ!+jV;1g}+E@B0W008cWPgMW_ diff --git a/my_dh_cli/src/my_dh_cli/__pycache__/cli.cpython-312.pyc b/my_dh_cli/src/my_dh_cli/__pycache__/cli.cpython-312.pyc deleted file mode 100644 index 8d23a93f7e28e8e181d948dc9fd9185805e89af3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1606 zcmah}&2QX96rZuZUhh}ZkVcRwDQ=ol7HKz7!X=afN|QrrtBMqm0+Ho-W_P{u+GfU1 zS!*M0L{ZaIb8DmuX?to=a^T8;fW&1*f>cZotq=!JECg}ri5c&jO{+?btoPo$H}C!4 zy!XaG7VIAg={muhk3Dc(S`kq!YEgUA-Fj;XZHhO7A zkO@Xv0m0$-jId%sBv_yZ*98a~kc@iV9<8bzOMQ;S2Dl55^tBjY8Ug_{ga(NB>taK! z#trE^0+cAjeyP?ahIQ(Hu#yd_`kZDkeB~&9Ls|s+RPg?XSVyX^&Mkhd^4qJrjZ_zt zl0#}tP0gCil%M;U)K!P+b|Xt4R0sBH?Aq!%!hCl4j7gAaj*C2By@s%W5DN zi^br@3y!~H;YGtCSbazB^4HX}+>ImomQI6o&kZ9rNT|fQPV;h@LKwRheGS`$7dA{7 zmT9bp5;iJMDElr;0f!Pb1DsC~-XOw9q zs$Oxi=hdhG11h+6uj1GRS2tGmGP}fdt*)Vp#*n8eF^=q*rs&qgOxGS2i5z1fJO%n0 zBTev|H2F9=@KDL!PTxvzecBppWq0P=DGZ#>S8Y*qJjCz}h8a>KWmZ!Nb^pKnjS zyScodd#|~0#A~S~-_P93Y_IRkw};OjC~rQ>3^gwwviRT{tDua(FsVnGBzbig2dP4# zH^G5WSPMqxZJpI#?dep<_t>cCg)nJStz_xtP^w_d0Vm~$Y{36lRFEHT_@On=c zfCiiT_}Q)t9zPSUD))G1AsA0Ndvrox490tnj^;H$CoBuQ@-?HYCLB>N-cU|%llSa5{f{`ZpI}HX#x5rwa?B@#2g@;n= z=EaSR?ZMMONw1J0?xbj1O*d^#%fp3yI>HJ?>XA@daU3g3AWX8HYV47OicZRY4clHQ zUDdrxC_65H8xbgvWa~AoY2Sb-bd=N+;CM$*>A5MoSHp=Z$E%JW*XlZ z-(20E**V^x`QQLv>Ifn{v&G^u)|!gg&dgtYcPIkkGUW6eCqo^XDLYvpWjBK##nGRo O Table: - """Filter table rows where column value exceeds threshold.""" - validate_columns(table, [column], raise_error=True) - return table.where(f"{column} > {threshold}") - - -def add_computed_columns(table: Table) -> Table: - """Add commonly used computed columns to a table.""" - validate_columns(table, ["Value"], raise_error=True) - return table.update( - [ - "DoubleValue = Value * 2", - "IsHigh = Value > 100", - ] - ) - - -def summarize_by_group(table: Table, group_col: str, value_col: str) -> Table: - """Create summary statistics grouped by a column.""" - validate_columns(table, [group_col, value_col], raise_error=True) - return table.agg_by( - [ - f"Sum = sum({value_col})", - f"Avg = avg({value_col})", - f"Count = count()", - ], - by=[group_col], - ) diff --git a/my_dh_library/build/lib/my_dh_library/utils.py b/my_dh_library/build/lib/my_dh_library/utils.py deleted file mode 100644 index f38c859..0000000 --- a/my_dh_library/build/lib/my_dh_library/utils.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Utility functions for working with Deephaven tables.""" - -from deephaven.table import Table - - -def validate_columns(table: Table, required_columns: list[str], raise_error: bool = False) -> bool: - """Check if table has all required columns. - - Args: - table: The table to validate - required_columns: List of column names that must be present - raise_error: If True, raises ValueError when columns are missing - - Returns: - True if all columns are present, False otherwise - - Raises: - ValueError: If raise_error is True and columns are missing - """ - table_columns = [col.name for col in table.columns] - missing = [col for col in required_columns if col not in table_columns] - - if missing: - if raise_error: - raise ValueError( - f"Column(s) {missing} not found in table. " - f"Available columns: {', '.join(table_columns)}" - ) - return False - return True - - -def get_table_info(table: Table) -> dict: - """Get basic information about a table.""" - return { - "num_rows": table.size, - "num_columns": len(table.columns), - "columns": [col.name for col in table.columns], - } diff --git a/my_dh_library/src/my_dh_library.egg-info/PKG-INFO b/my_dh_library/src/my_dh_library.egg-info/PKG-INFO deleted file mode 100644 index a450448..0000000 --- a/my_dh_library/src/my_dh_library.egg-info/PKG-INFO +++ /dev/null @@ -1,63 +0,0 @@ -Metadata-Version: 2.4 -Name: my_dh_library -Version: 0.1.0 -Summary: Reusable Deephaven query functions -Requires-Python: >=3.8 -Description-Content-Type: text/markdown -Requires-Dist: deephaven-server>=0.35.0 - -# My Deephaven Library - -A library-only package providing reusable Deephaven query functions. This package contains no CLI tools - it's designed to be imported and used as a library in other Python projects. - -## Installation - -```shell -pip install . -``` - -Or in editable mode for development: - -```shell -pip install -e . -``` - -## Usage - -> [!NOTE] -> All Deephaven functionality requires a running server. Start the server before importing Deephaven modules. - -Import and use the library functions in your Python code: - -```python -# Start the Deephaven server -from deephaven_server import Server -server = Server(port=10000, jvm_args=["-Xmx4g"]) -server.start() - -# Now use the library functions -from my_dh_library.queries import filter_by_threshold, add_computed_columns -from deephaven import read_csv - -data = read_csv("data/sample.csv") -filtered = filter_by_threshold(data, "Score", 75.0) -enhanced = add_computed_columns(filtered) -``` - -## Available Functions - -### Query Functions (`my_dh_library.queries`) - -- `filter_by_threshold(table, column, threshold)` - Filter table rows where column value exceeds threshold -- `add_computed_columns(table)` - Add commonly used computed columns to a table -- `summarize_by_group(table, group_col, value_col)` - Create summary statistics grouped by a column - -### Utility Functions (`my_dh_library.utils`) - -- `validate_columns(table, required_columns)` - Check if table has all required columns -- `get_table_info(table)` - Get basic information about a table - -## Requirements - -- Python 3.8 or later -- Deephaven Server 0.35.0 or later diff --git a/my_dh_library/src/my_dh_library.egg-info/SOURCES.txt b/my_dh_library/src/my_dh_library.egg-info/SOURCES.txt deleted file mode 100644 index 29745ee..0000000 --- a/my_dh_library/src/my_dh_library.egg-info/SOURCES.txt +++ /dev/null @@ -1,10 +0,0 @@ -README.md -pyproject.toml -src/my_dh_library/__init__.py -src/my_dh_library/queries.py -src/my_dh_library/utils.py -src/my_dh_library.egg-info/PKG-INFO -src/my_dh_library.egg-info/SOURCES.txt -src/my_dh_library.egg-info/dependency_links.txt -src/my_dh_library.egg-info/requires.txt -src/my_dh_library.egg-info/top_level.txt \ No newline at end of file diff --git a/my_dh_library/src/my_dh_library.egg-info/dependency_links.txt b/my_dh_library/src/my_dh_library.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789..0000000 --- a/my_dh_library/src/my_dh_library.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/my_dh_library/src/my_dh_library.egg-info/requires.txt b/my_dh_library/src/my_dh_library.egg-info/requires.txt deleted file mode 100644 index b68ceb0..0000000 --- a/my_dh_library/src/my_dh_library.egg-info/requires.txt +++ /dev/null @@ -1 +0,0 @@ -deephaven-server>=0.35.0 diff --git a/my_dh_library/src/my_dh_library.egg-info/top_level.txt b/my_dh_library/src/my_dh_library.egg-info/top_level.txt deleted file mode 100644 index e663cac..0000000 --- a/my_dh_library/src/my_dh_library.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -my_dh_library diff --git a/my_dh_library/src/my_dh_library/__pycache__/__init__.cpython-312.pyc b/my_dh_library/src/my_dh_library/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index b37569a3e1e4e05ff48dffac6d0a09febdba9acd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 427 zcmZ8cu};G<6m*(|qE=-9DlujxLzA*2CRSi#Aj^{*o5o9bn5Bir diff --git a/my_dh_library/src/my_dh_library/__pycache__/queries.cpython-312.pyc b/my_dh_library/src/my_dh_library/__pycache__/queries.cpython-312.pyc deleted file mode 100644 index 7bf4d85a749901ec5d4235a21f824fd200f8a80b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1594 zcma)6PiP!f7=QD3|0lu5G*X3UYb9IkZW4qFiGk8uS`-nRR-4N@JMU$7>demi-c0HY zd+;D)PkQS@DeN&NIppZE$AXuD1nNLRP!GKgtfzqB_ulL-jTe0|-=8<%y!p-d`+f6s ztyV^`{$BgeJ)gA}Mq zid5Y$lG3;ORzdEITqkA7i!!f3T@|jpt_J(1c&0v^ocokU+-iFiU!}Cyv36(|e-%*{ zYIFv>fDM^cl~ zFk)R>6DN(A0bb;zkH9=Yp*U8qKI$uuLX>@VZ(OgSKDztzvy)2YA#|j#K17eSz47xm zQC&|iU30yVGAsdM7Tn|bUWYP@b7&Z#M-$4XjZmdk02G8V8z!yCA+ zGJ!QUEfMJY}KX2aRl<_9@+`Cq%N5e2~l1Wo9_TsP;bT9TS`%BAl zyH3-OTcp$S+%~gV+~myu*N$MprM%IL(^K2>b6V}VHO0zi;k5z;zX0YN^oKtGahVWkx9ssF51C1t*)V0w9vpSAz(o zB)9PUIQJLuYBGO=Uw56(bZ-N%EH6(8GNYc*V_lI9aq*3q&45kkEkZ!1zI^>euWTM} z)!^dqfSGjgv%!sncaQYi-dWE^wkh(ug6XE+f;?#E|S4_Eb0@7RbJsG_EiT4Zu9| zeGnDqp%uD3bZw3$R%m@YhL-13&=@znd^7T)q43bsu~JXUmvSKhwMbyM zTIuX`3>)%~zyv9&5rwYD^V!yT$Wmk53#?EMmehuX`4~yI<+mIc*ZqJ*9^GJP;RGQt zZiC4*MNxi5^*_;@&(XQe)Rfghlp%1twsb#H>7E)Rv6)qiia9hg1a9~INVsgFte69+ S0(YQE_at{ujlaaE1o01GYM2}V From d5c749d91e9d182505d676b88eaffacb83ed0fb5 Mon Sep 17 00:00:00 2001 From: elijahpetty Date: Thu, 27 Aug 2026 16:59:20 -0500 Subject: [PATCH 05/13] wip --- my_dh_cli/src/my_dh_cli/cli.py | 5 +- my_dh_library/src/my_dh_library/queries.py | 7 +- my_dh_toolkit/src/my_dh_toolkit/cli.py | 5 +- my_dh_toolkit/src/my_dh_toolkit/processor.py | 9 +- my_dh_toolkit/src/my_dh_toolkit/queries.py | 7 +- setuptools-deployment.md | 774 +++++++++++++++++++ 6 files changed, 793 insertions(+), 14 deletions(-) create mode 100644 setuptools-deployment.md diff --git a/my_dh_cli/src/my_dh_cli/cli.py b/my_dh_cli/src/my_dh_cli/cli.py index d074cfd..3e05e1c 100644 --- a/my_dh_cli/src/my_dh_cli/cli.py +++ b/my_dh_cli/src/my_dh_cli/cli.py @@ -21,10 +21,11 @@ def my_dh_query(input_file: str, verbose: bool = False): except Exception as e: raise click.ClickException(f"Failed to read CSV file '{input_file}': {e}") - if "Score" not in source.columns: + column_names = [col.name for col in source.columns] + if "Score" not in column_names: raise click.ClickException( f"File '{input_path.name}' is missing required column 'Score'. " - f"Available columns: {', '.join(source.columns)}" + f"Available columns: {', '.join(column_names)}" ) result = source.update(formulas=["DoubleScore = Score * 2"]) diff --git a/my_dh_library/src/my_dh_library/queries.py b/my_dh_library/src/my_dh_library/queries.py index 6471685..eb0adfc 100644 --- a/my_dh_library/src/my_dh_library/queries.py +++ b/my_dh_library/src/my_dh_library/queries.py @@ -1,6 +1,7 @@ """Reusable Deephaven query functions.""" from deephaven.table import Table +from deephaven import agg from .utils import validate_columns @@ -26,9 +27,9 @@ def summarize_by_group(table: Table, group_col: str, value_col: str) -> Table: validate_columns(table, [group_col, value_col], raise_error=True) return table.agg_by( [ - f"Sum = sum({value_col})", - f"Avg = avg({value_col})", - f"Count = count()", + agg.sum_(f"Sum = {value_col}"), + agg.avg(f"Avg = {value_col}"), + agg.count_("Count"), ], by=[group_col], ) diff --git a/my_dh_toolkit/src/my_dh_toolkit/cli.py b/my_dh_toolkit/src/my_dh_toolkit/cli.py index d074cfd..3e05e1c 100644 --- a/my_dh_toolkit/src/my_dh_toolkit/cli.py +++ b/my_dh_toolkit/src/my_dh_toolkit/cli.py @@ -21,10 +21,11 @@ def my_dh_query(input_file: str, verbose: bool = False): except Exception as e: raise click.ClickException(f"Failed to read CSV file '{input_file}': {e}") - if "Score" not in source.columns: + column_names = [col.name for col in source.columns] + if "Score" not in column_names: raise click.ClickException( f"File '{input_path.name}' is missing required column 'Score'. " - f"Available columns: {', '.join(source.columns)}" + f"Available columns: {', '.join(column_names)}" ) result = source.update(formulas=["DoubleScore = Score * 2"]) diff --git a/my_dh_toolkit/src/my_dh_toolkit/processor.py b/my_dh_toolkit/src/my_dh_toolkit/processor.py index 1e50cea..526c766 100644 --- a/my_dh_toolkit/src/my_dh_toolkit/processor.py +++ b/my_dh_toolkit/src/my_dh_toolkit/processor.py @@ -4,7 +4,7 @@ def batch_process(directory: str, output_dir: str, verbose: bool = False) -> None: """Process multiple CSV files from a directory.""" - from deephaven import read_csv + from deephaven import read_csv, write_csv input_path = Path(directory) output_path = Path(output_dir) @@ -32,17 +32,18 @@ def batch_process(directory: str, output_dir: str, verbose: bool = False) -> Non table = read_csv(str(csv_file)) - if "Score" not in table.columns: + column_names = [col.name for col in table.columns] + if "Score" not in column_names: raise click.ClickException( f"File '{csv_file.name}' is missing required column 'Score'. " - f"Available columns: {', '.join(table.columns)}" + f"Available columns: {', '.join(column_names)}" ) processed = table.update(formulas=["ProcessedScore = Score * 2"]) output_file = output_path / f"processed_{csv_file.name}" try: - processed.to_csv(str(output_file)) + write_csv(processed, str(output_file)) except Exception as e: raise click.ClickException(f"Failed to write output file '{output_file}': {e}") diff --git a/my_dh_toolkit/src/my_dh_toolkit/queries.py b/my_dh_toolkit/src/my_dh_toolkit/queries.py index 6471685..eb0adfc 100644 --- a/my_dh_toolkit/src/my_dh_toolkit/queries.py +++ b/my_dh_toolkit/src/my_dh_toolkit/queries.py @@ -1,6 +1,7 @@ """Reusable Deephaven query functions.""" from deephaven.table import Table +from deephaven import agg from .utils import validate_columns @@ -26,9 +27,9 @@ def summarize_by_group(table: Table, group_col: str, value_col: str) -> Table: validate_columns(table, [group_col, value_col], raise_error=True) return table.agg_by( [ - f"Sum = sum({value_col})", - f"Avg = avg({value_col})", - f"Count = count()", + agg.sum_(f"Sum = {value_col}"), + agg.avg(f"Avg = {value_col}"), + agg.count_("Count"), ], by=[group_col], ) diff --git a/setuptools-deployment.md b/setuptools-deployment.md new file mode 100644 index 0000000..8af03a6 --- /dev/null +++ b/setuptools-deployment.md @@ -0,0 +1,774 @@ +--- +title: Packaging custom code and dependencies +sidebar_label: Python packaging +--- + +[Python packaging](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/) enables you to create distributable packages containing custom code, command-line tools, and managed dependencies. Deephaven's pip-installable packages integrate seamlessly with modern Python packaging tools, allowing you to build reusable libraries and executable scripts that leverage Deephaven's query engine. This guide walks through the concepts and patterns for packaging Deephaven-based Python projects. + +Python packaging with [`pyproject.toml`](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/) provides: + +- **Reusable libraries** - Package query functions and utilities for import by other projects. +- **Command-line tools** - Build executable scripts with entry point definitions. +- **Dependency management** - Automatically install Deephaven and required packages. +- **Distribution** - Share code as wheel archives via PyPI or direct distribution. +- **Version control** - Specify compatible dependency versions for reproducible installations. + +## Example repository + +The examples in this guide use the [deephaven-python-packaging](https://github.com/deephaven-examples/deephaven-python-packaging) repository. It demonstrates three complete packaging scenarios with working code, sample data, and comprehensive documentation. + +To explore the examples, clone the repository: + +```bash +git clone https://github.com/deephaven-examples/deephaven-python-packaging.git +cd deephaven-python-packaging +``` + +The repository contains three example packages: + +- `my_dh_library/` - Library-only package with reusable query functions. +- `my_dh_cli/` - CLI-only package with command-line tools. +- `my_dh_toolkit/` - Combined package with both library and CLI functionality. + +## Package structure + +Modern Python packages use the **src-layout**, which is the recommended structure by the [Python Packaging Authority](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/). This layout keeps source code separate from tests and configuration files: + +``` +my_dh_project/ +├── src/ +│ └── my_dh_package/ +│ ├── __init__.py +│ ├── queries.py +│ └── utils.py +├── pyproject.toml +└── README.md +``` + +### Key components + +- **`src/`** - Source directory containing the package code. +- **`my_dh_package/`** - The Python package (directory name used in imports). +- **`__init__.py`** - Makes the directory importable and can export public API. +- **`pyproject.toml`** - Defines package metadata, dependencies, and entry points. +- **Module files** - Python files containing your functions and classes. + +The package name under `src/` determines how users import your code. For example, with `src/my_dh_library/`, users import via `from my_dh_library import ...`. + +## Packaging scenarios + +Different projects have different needs. The example repository demonstrates three common scenarios: + +### Library-only package + +Package reusable code without CLI tools. Other projects import your modules. + +**Structure:** + +``` +my_dh_library/ +├── src/ +│ └── my_dh_library/ +│ ├── __init__.py +│ ├── queries.py +│ └── utils.py +├── pyproject.toml +└── README.md +``` + +**Usage:** + +```python +from my_dh_library.queries import filter_by_threshold, add_computed_columns +from deephaven import read_csv + +data = read_csv("data.csv") +filtered = filter_by_threshold(data, "Score", 75.0) +``` + +**When to use:** + +- Creating reusable utilities for other projects. +- No command-line interface needed. +- Code will be imported, not executed directly. + +### CLI-only package + +Package executable command-line tools without exposing library code. + +**Structure:** + +``` +my_dh_cli/ +├── src/ +│ └── my_dh_cli/ +│ ├── __init__.py +│ ├── __main__.py +│ └── cli.py +├── pyproject.toml +└── README.md +``` + +**Usage:** + +```python +# CLI functions are used within a Python session +from my_dh_cli.cli import my_dh_query +result = my_dh_query("data.csv", verbose=True) +``` + +**When to use:** + +- Building command-line tools for data processing +- Want clean function interfaces +- No library code to expose to other projects + +### Combined package + +Package both reusable library code and command-line tools. + +**Structure:** + +``` +my_dh_toolkit/ +├── src/ +│ └── my_dh_toolkit/ +│ ├── __init__.py +│ ├── __main__.py +│ ├── cli.py +│ ├── processor.py +│ ├── queries.py +│ └── utils.py +├── pyproject.toml +└── README.md +``` + +**Usage:** + +```python +# As a library +from my_dh_toolkit.queries import filter_by_threshold +from my_dh_toolkit import my_dh_query + +# Use library functions +data = read_csv("data.csv") +filtered = filter_by_threshold(data, "Score", 75.0) + +# Or use CLI functions +result = my_dh_query("data.csv", verbose=True) +``` + +**When to use:** + +- Need both library and CLI functionality +- Want to provide multiple interfaces to the same code +- Library functions are useful independently + +## Create a new package + +
+Step-by-step instructions for creating packages from scratch + +This section walks through creating each type of package from scratch. + +### Create a library-only package + +Create the directory structure: + +```bash +mkdir -p my_dh_library/src/my_dh_library +cd my_dh_library +``` + +Create `pyproject.toml`: + +```toml +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "my_dh_library" +version = "0.1.0" +description = "Reusable Deephaven query functions" +readme = "README.md" +requires-python = ">=3.8" +dependencies = [ + "deephaven-server>=0.35.0", +] + +[tool.setuptools.packages.find] +where = ["src"] +``` + +Create `src/my_dh_library/__init__.py`: + +```python +"""My Deephaven package for data processing.""" + +__version__ = "0.1.0" + +from my_dh_library.queries import filter_by_threshold, add_computed_columns, summarize_by_group + +__all__ = ["filter_by_threshold", "add_computed_columns", "summarize_by_group"] +``` + +Create `src/my_dh_library/utils.py`: + +```python +"""Utility functions for working with Deephaven tables.""" + +from deephaven.table import Table + + +def validate_columns(table: Table, required_columns: list[str], raise_error: bool = False) -> bool: + """Check if table has all required columns. + + Args: + table: The table to validate + required_columns: List of column names that must be present + raise_error: If True, raises ValueError when columns are missing + + Returns: + True if all columns are present, False otherwise + + Raises: + ValueError: If raise_error is True and columns are missing + """ + table_columns = [col.name for col in table.columns] + missing = [col for col in required_columns if col not in table_columns] + + if missing: + if raise_error: + raise ValueError( + f"Column(s) {missing} not found in table. " + f"Available columns: {', '.join(table_columns)}" + ) + return False + return True + + +def get_table_info(table: Table) -> dict: + """Get basic information about a table.""" + return { + "num_rows": table.size, + "num_columns": len(table.columns), + "columns": [col.name for col in table.columns], + } +``` + +Create `src/my_dh_library/queries.py`: + +```python +"""Reusable Deephaven query functions.""" + +from deephaven.table import Table +from deephaven import agg +from .utils import validate_columns + + +def filter_by_threshold(table: Table, column: str, threshold: float) -> Table: + """Filter table rows where column value exceeds threshold.""" + validate_columns(table, [column], raise_error=True) + return table.where(f"{column} > {threshold}") + + +def add_computed_columns(table: Table) -> Table: + """Add commonly used computed columns to a table.""" + validate_columns(table, ["Value"], raise_error=True) + return table.update( + [ + "DoubleValue = Value * 2", + "IsHigh = Value > 100", + ] + ) + + +def summarize_by_group(table: Table, group_col: str, value_col: str) -> Table: + """Create summary statistics grouped by a column.""" + validate_columns(table, [group_col, value_col], raise_error=True) + return table.agg_by( + [ + agg.sum_(f"Sum = {value_col}"), + agg.avg(f"Avg = {value_col}"), + agg.count_("Count"), + ], + by=[group_col], + ) +``` + +Create `README.md` with installation and usage instructions. + +### Create a CLI-only package + +Create the directory structure: + +```bash +mkdir -p my_dh_cli/src/my_dh_cli +cd my_dh_cli +``` + +Create `pyproject.toml`: + +```toml +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "my_dh_cli" +version = "0.1.0" +description = "Command-line tool for data processing" +readme = "README.md" +requires-python = ">=3.8" +dependencies = [ + "deephaven-server>=0.35.0", + "click>=8.0.0", +] + +[project.scripts] +my-dh-query = "my_dh_cli.cli:app" + +[tool.setuptools.packages.find] +where = ["src"] +``` + +Create `src/my_dh_cli/__init__.py`: + +```python +"""My Deephaven package for data processing.""" + +__version__ = "0.1.0" +``` + +Create `src/my_dh_cli/__main__.py`: + +```python +from my_dh_cli.cli import app + +if __name__ == "__main__": + app() +``` + +Create `src/my_dh_cli/cli.py`: + +```python +import click + + +def my_dh_query(input_file: str, verbose: bool = False): + """Read a CSV file and perform a simple query operation on the data.""" + from deephaven import read_csv + from pathlib import Path + + input_path = Path(input_file) + + if not input_path.exists(): + raise click.ClickException(f"Input file does not exist: '{input_path}'") + if not input_path.is_file(): + raise click.ClickException(f"Input path is not a file: '{input_path}'") + + if verbose: + click.echo(f"Processing {input_file}...") + + try: + source = read_csv(input_file) + except Exception as e: + raise click.ClickException(f"Failed to read CSV file '{input_file}': {e}") + + column_names = [col.name for col in source.columns] + if "Score" not in column_names: + raise click.ClickException( + f"File '{input_path.name}' is missing required column 'Score'. " + f"Available columns: {', '.join(column_names)}" + ) + + result = source.update(formulas=["DoubleScore = Score * 2"]) + + if verbose: + click.echo(f"Processed {result.size} rows") + + return result + + +@click.command() +@click.argument("input_file", type=click.Path(exists=True)) +@click.option("--verbose", "-v", is_flag=True, help="Enable verbose output") +def app(input_file: str, verbose: bool) -> None: + """Process data with Deephaven.""" + result = my_dh_query(input_file, verbose) + click.echo("Processing complete!") + + +if __name__ == "__main__": + app() +``` + +Create `README.md` with installation and usage instructions. + +### Create a combined package + +Create the directory structure: + +```bash +mkdir -p my_dh_toolkit/src/my_dh_toolkit +cd my_dh_toolkit +``` + +Create `pyproject.toml`: + +```toml +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "my_dh_toolkit" +version = "0.1.0" +description = "Deephaven library and CLI tools" +readme = "README.md" +requires-python = ">=3.8" +dependencies = [ + "deephaven-server>=0.35.0", + "click>=8.0.0", +] + +[project.scripts] +my-dh-query = "my_dh_toolkit.cli:app" +my-dh-process = "my_dh_toolkit.processor:process" + +[tool.setuptools.packages.find] +where = ["src"] +``` + +Create `src/my_dh_toolkit/__init__.py`: + +```python +"""My Deephaven package for data processing.""" + +__version__ = "0.1.0" + +from my_dh_toolkit.cli import my_dh_query +from my_dh_toolkit.processor import batch_process + +__all__ = ["my_dh_query", "batch_process"] +``` + +Create `src/my_dh_toolkit/__main__.py`: + +```python +from my_dh_toolkit.cli import app + +if __name__ == "__main__": + app() +``` + +Create the library modules (`queries.py`, `utils.py`) using the same code as the library-only package. + +Create `src/my_dh_toolkit/cli.py` using the same code as the CLI-only package. + +Create `src/my_dh_toolkit/processor.py`: + +```python +import click +from pathlib import Path + + +def batch_process(directory: str, output_dir: str, verbose: bool = False) -> None: + """Process multiple CSV files from a directory.""" + from deephaven import read_csv, write_csv + + input_path = Path(directory) + output_path = Path(output_dir) + + if not input_path.exists(): + raise click.ClickException(f"Input directory does not exist: '{input_path}'") + if not input_path.is_dir(): + raise click.ClickException(f"Input path is not a directory: '{input_path}'") + + try: + output_path.mkdir(parents=True, exist_ok=True) + except PermissionError: + raise click.ClickException(f"Permission denied: Cannot create output directory '{output_path}'") + except OSError as e: + raise click.ClickException(f"Failed to create output directory '{output_path}': {e}") + + csv_files = list(input_path.glob("*.csv")) + + if verbose: + click.echo(f"Found {len(csv_files)} CSV files to process") + + for csv_file in csv_files: + if verbose: + click.echo(f"Processing {csv_file.name}...") + + table = read_csv(str(csv_file)) + + column_names = [col.name for col in table.columns] + if "Score" not in column_names: + raise click.ClickException( + f"File '{csv_file.name}' is missing required column 'Score'. " + f"Available columns: {', '.join(column_names)}" + ) + + processed = table.update(formulas=["ProcessedScore = Score * 2"]) + + output_file = output_path / f"processed_{csv_file.name}" + try: + write_csv(processed, str(output_file)) + except Exception as e: + raise click.ClickException(f"Failed to write output file '{output_file}': {e}") + + if verbose: + click.echo(f" Processed {processed.size} rows -> {output_file.name}") + + +@click.command() +@click.argument("directory", type=click.Path(exists=True, file_okay=False)) +@click.option("--output", "-o", default="./output", help="Output directory") +@click.option("--verbose", "-v", is_flag=True, help="Enable verbose output") +def process(directory: str, output: str, verbose: bool) -> None: + """Batch process CSV files with Deephaven.""" + batch_process(directory, output, verbose) + click.echo("Batch processing complete!") + + +if __name__ == "__main__": + process() +``` + +Create `README.md` with installation and usage instructions. + +
+ +## Configure `pyproject.toml` + +The [`pyproject.toml`](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/) file defines your package configuration. + +### Configuration options + +Here's a detailed breakdown of `pyproject.toml` for a library-only package: + +```toml +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "my_dh_library" +version = "0.1.0" +description = "Reusable Deephaven query functions" +readme = "README.md" +requires-python = ">=3.8" +dependencies = [ + "deephaven-server>=0.35.0", +] + +[tool.setuptools.packages.find] +where = ["src"] +``` + +### Key sections + +- **`[build-system]`** - Specifies setuptools as the build backend +- **`[project]`** - Package metadata and dependencies +- **`name`** - Project name (used for `pip install`) +- **`dependencies`** - Required packages installed automatically +- **`[tool.setuptools.packages.find]`** - Tells setuptools to find packages in `src/` + +For CLI packages, add a `[project.scripts]` section: + +```toml +[project.scripts] +my-dh-query = "my_dh_cli.cli:app" +``` + +This creates a command-line entry point that calls the `app` function from `my_dh_cli.cli`. + +## Managing dependencies + +Dependencies are specified in the `dependencies` field: + +```toml +[project] +dependencies = [ + "deephaven-server>=0.35.0", + "click>=8.0.0", + "pandas>=2.0.0", +] +``` + +### Version constraints + +Use version specifiers to control which versions are acceptable: + +- `>=0.35.0` - Minimum version (0.35.0 or higher) +- `>=2.0.0,<3.0.0` - Version range (2.x only) +- `~=1.24.0` - Compatible release (>=1.24.0, <1.25.0) +- `==1.0.0` - Exact version (not recommended for libraries) + +### Optional dependencies + +Define optional feature sets that users can install separately: + +```toml +[project.optional-dependencies] +visualization = [ + "matplotlib>=3.7.0", + "seaborn>=0.12.0", +] +dev = [ + "pytest>=7.0.0", + "black>=23.0.0", +] +``` + +Users can install optional dependencies: + +```bash +pip install my_dh_library[visualization] +pip install my_dh_library[visualization,dev] +``` + +## Installation and usage + +### Install a package + +Install from source in editable mode for development: + +```bash +cd my_dh_library +pip install -e . +``` + +Or install normally: + +```bash +pip install . +``` + +### Use a library package + +After installation, import and use the library functions: + +```python +# Start the Deephaven server +from deephaven_server import Server +server = Server(port=10000, jvm_args=["-Xmx4g"]) +server.start() + +# Import and use library functions +from my_dh_library.queries import filter_by_threshold +from deephaven import read_csv + +data = read_csv("data.csv") +filtered = filter_by_threshold(data, "Score", 75.0) +``` + +> [!NOTE] +> All Deephaven functionality requires a running server. Start the server before importing Deephaven modules. + +### Use CLI functions + +CLI functions must be used within the same Python session as the server: + +```python +# Start the Deephaven server +from deephaven_server import Server +server = Server(port=10000, jvm_args=["-Xmx4g"]) +server.start() + +# Use CLI functions +from my_dh_cli.cli import my_dh_query +result = my_dh_query("data.csv", verbose=True) +``` + +## Building and distributing + +Build a distributable wheel: + +```bash +cd my_dh_library +pip install build +python -m build +``` + +This creates a `.whl` file in `dist/` that can be: + +- Installed locally: `pip install dist/my_dh_library-0.1.0-py3-none-any.whl` +- Distributed to others +- Published to PyPI: `python -m twine upload dist/*` + +## Best practices + +### Package structure + +- Use the src-layout for all packages +- Keep package names lowercase with underscores +- Match the package directory name to the import name +- Include `__init__.py` in all package directories + +### Dependencies + +- Specify minimum versions for Deephaven and critical dependencies +- Use version ranges for flexibility +- Group related optional dependencies +- Document any system-level dependencies + +### Documentation + +- Include a comprehensive README.md +- Document all public functions and classes +- Provide usage examples +- Explain server initialization requirements + +### Testing + +- Write tests for all public functions +- Test with different Deephaven versions +- Include sample data for testing +- Document how to run tests + +## Server initialization + +Deephaven requires a running server before using any Deephaven functionality. The server must be initialized in the same Python process that uses Deephaven: + +```python +from deephaven_server import Server + +# Initialize and start the server +server = Server(port=10000, jvm_args=["-Xmx4g"]) +server.start() + +# Now you can import and use Deephaven +from deephaven import read_csv +data = read_csv("data.csv") +``` + +### Key points + +- Each Python process has its own JVM +- Starting a server in one terminal doesn't help another terminal +- CLI tools must run in the same session as the server +- The server uses approximately 4GB of memory by default (configurable via `jvm_args`) + +## Next steps + +The [deephaven-python-packaging](https://github.com/deephaven-examples/deephaven-python-packaging) repository provides complete, working examples of all three packaging scenarios. Clone the repository and explore the examples to see how to structure your own Deephaven packages. + +Each example includes: + +- Complete source code +- Configured `pyproject.toml` +- Sample data files +- Comprehensive README +- Usage examples + +## Related documentation + +- [Install and use Python packages](https://deephaven.io/core/docs/how-to-guides/install-and-use-python-packages/) +- [Use the Deephaven Python package](https://deephaven.io/core/docs/how-to-guides/deephaven-python-package/) +- [Python Packaging User Guide](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/) +- [Creating command-line tools](https://packaging.python.org/en/latest/guides/creating-command-line-tools/) +- [Setuptools documentation](https://setuptools.pypa.io/) +- [Click documentation](https://click.palletsprojects.com/) From 99946fbbbae5febc05059cd4b090d7d812bb492d Mon Sep 17 00:00:00 2001 From: elijahpetty Date: Mon, 31 Aug 2026 16:37:44 -0500 Subject: [PATCH 06/13] wip --- README.md | 36 ++++----- my_dh_cli/README.md | 10 +-- my_dh_cli/src/my_dh_cli/cli.py | 4 + my_dh_toolkit/README.md | 16 ++-- my_dh_toolkit/src/my_dh_toolkit/cli.py | 4 + my_dh_toolkit/src/my_dh_toolkit/processor.py | 4 + setuptools-deployment.md | 79 ++++++++++++-------- 7 files changed, 89 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index dbaf300..84ecf7b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Python Packaging with Deephaven -This repository demonstrates how to create and deploy Python packages that use Deephaven. It shows three complete packaging scenarios following the official [Python Packaging User Guide](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/) recommendations. +This repository demonstrates how to create and deploy Python packages that use Deephaven Community Core. It shows three complete packaging scenarios following the official [Python Packaging User Guide](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/) recommendations. This example accompanies the [Packaging custom code and dependencies](https://deephaven.io/core/docs/how-to-guides/sysadmin/setuptools-deployment/) guide in the Deephaven documentation. @@ -59,10 +59,8 @@ my_dh_cli/ ``` **Usage:** -```python -# Use within a Python session with server running -from my_dh_cli.cli import my_dh_query -result = my_dh_query("input_data.csv", verbose=True) +```shell +my-dh-query input_data.csv --verbose ``` ### 3. Combined package (`my_dh_toolkit/`) @@ -89,11 +87,10 @@ my_dh_toolkit/ from my_dh_toolkit.queries import filter_by_threshold ``` -```python -# As CLI functions (within Python session) -from my_dh_toolkit import my_dh_query, batch_process -result = my_dh_query("input_data.csv", verbose=True) -batch_process("data/", "results/", verbose=True) +```shell +# As CLI commands +my-dh-query input_data.csv --verbose +my-dh-process data/ --output results/ --verbose ``` ## Quick start @@ -131,16 +128,15 @@ filtered = filter_by_threshold(data, "Score", 75.0) ### Try the CLI-only package -> [!NOTE] -> CLI tools require a Deephaven server running in the same Python process. The examples below show how to use the CLI functions within a Python session where the server is already started. True standalone CLI commands (run from a separate terminal) are not practical with Deephaven due to JVM initialization requirements. +Install the package, then run the installed command directly from a terminal. `my-dh-query` starts its own Deephaven server, so no separate session setup is needed: ```shell cd my_dh_cli pip install -e . -python +my-dh-query ../data/sample.csv --verbose ``` -Then in Python: +The underlying `my_dh_query()` function is also importable, so you can call it directly within a Python session that already has a server running (useful when composing it with other Deephaven code): ```python # Start the Deephaven server @@ -148,7 +144,7 @@ from deephaven_server import Server server = Server(port=10000, jvm_args=["-Xmx4g"]) server.start() -# Now use the CLI function +# Call the underlying function directly from my_dh_cli.cli import my_dh_query result = my_dh_query("../data/sample.csv", verbose=True) print(f"Processed {result.size} rows") @@ -156,16 +152,16 @@ print(f"Processed {result.size} rows") ### Try the combined package -> [!NOTE] -> Like the CLI-only package, the CLI commands require a Deephaven server in the same Python process. Use the library functions within a Python session. +Install the package, then run the installed commands directly from a terminal. Both `my-dh-query` and `my-dh-process` start their own Deephaven server: ```shell cd my_dh_toolkit pip install -e . -python +my-dh-query ../data/sample.csv --verbose +my-dh-process ../data/batch --output ./output --verbose ``` -Then in Python: +The library functions are also importable for use within a Python session that already has a server running: ```python # Start the Deephaven server @@ -180,7 +176,7 @@ from deephaven import read_csv data = read_csv("../data/sample.csv") filtered = filter_by_threshold(data, "Score", 75.0) -# Or use the CLI functions +# Or call the underlying CLI functions directly from my_dh_toolkit import my_dh_query, batch_process result = my_dh_query("../data/sample.csv", verbose=True) batch_process("../data/batch/", "./output", verbose=True) diff --git a/my_dh_cli/README.md b/my_dh_cli/README.md index 66e3a4a..709a274 100644 --- a/my_dh_cli/README.md +++ b/my_dh_cli/README.md @@ -16,15 +16,13 @@ pip install -e . ## Usage -> [!NOTE] -> CLI functions require a Deephaven server running in the same Python process. Use the functions within a Python session where the server is already started. +Run the installed command directly from a terminal. `my-dh-query` starts its own Deephaven server, so no separate session setup is needed: ```shell -pip install -e . -python +my-dh-query data/sample.csv --verbose ``` -Then in Python: +The underlying `my_dh_query()` function is also importable, so you can call it directly within a Python session that already has a server running: ```python # Start the Deephaven server @@ -32,7 +30,7 @@ from deephaven_server import Server server = Server(port=10000, jvm_args=["-Xmx4g"]) server.start() -# Now use the CLI function +# Call the underlying function directly from my_dh_cli.cli import my_dh_query result = my_dh_query("data/sample.csv", verbose=True) print(f"Processed {result.size} rows") diff --git a/my_dh_cli/src/my_dh_cli/cli.py b/my_dh_cli/src/my_dh_cli/cli.py index 3e05e1c..640248e 100644 --- a/my_dh_cli/src/my_dh_cli/cli.py +++ b/my_dh_cli/src/my_dh_cli/cli.py @@ -41,6 +41,10 @@ def my_dh_query(input_file: str, verbose: bool = False): @click.option("--verbose", "-v", is_flag=True, help="Enable verbose output") def app(input_file: str, verbose: bool) -> None: """Process data with Deephaven.""" + from deephaven_server import Server + + Server(port=10000, jvm_args=["-Xmx4g"]).start() + result = my_dh_query(input_file, verbose) click.echo("Processing complete!") diff --git a/my_dh_toolkit/README.md b/my_dh_toolkit/README.md index bb91728..6bddf5f 100644 --- a/my_dh_toolkit/README.md +++ b/my_dh_toolkit/README.md @@ -1,6 +1,6 @@ # My Deephaven Toolkit -A combined package providing both reusable library code and command-line functions for Deephaven. This package can be used as both a library (imported in Python code) and as CLI functions (called within a Python session). +A combined package providing both reusable library code and command-line tools for Deephaven. This package can be used as both a library (imported in Python code) and as standalone CLI commands. ## Installation @@ -39,10 +39,16 @@ filtered = filter_by_threshold(data, "Score", 75.0) result = my_dh_query("data/sample.csv", verbose=True) ``` -## Usage as CLI Functions +## Usage as CLI Commands -> [!NOTE] -> CLI functions require a Deephaven server running in the same Python process. Use them within a Python session where the server is already started. +Run the installed commands directly from a terminal. Both `my-dh-query` and `my-dh-process` start their own Deephaven server: + +```shell +my-dh-query data/sample.csv --verbose +my-dh-process data/batch --output ./output --verbose +``` + +The underlying functions are also importable, so you can call them directly within a Python session that already has a server running: ```python # Start the Deephaven server @@ -50,7 +56,7 @@ from deephaven_server import Server server = Server(port=10000, jvm_args=["-Xmx4g"]) server.start() -# Use the CLI functions +# Call the underlying functions directly from my_dh_toolkit import my_dh_query, batch_process result = my_dh_query("data/sample.csv", verbose=True) batch_process("data/batch/", "./output", verbose=True) diff --git a/my_dh_toolkit/src/my_dh_toolkit/cli.py b/my_dh_toolkit/src/my_dh_toolkit/cli.py index 3e05e1c..640248e 100644 --- a/my_dh_toolkit/src/my_dh_toolkit/cli.py +++ b/my_dh_toolkit/src/my_dh_toolkit/cli.py @@ -41,6 +41,10 @@ def my_dh_query(input_file: str, verbose: bool = False): @click.option("--verbose", "-v", is_flag=True, help="Enable verbose output") def app(input_file: str, verbose: bool) -> None: """Process data with Deephaven.""" + from deephaven_server import Server + + Server(port=10000, jvm_args=["-Xmx4g"]).start() + result = my_dh_query(input_file, verbose) click.echo("Processing complete!") diff --git a/my_dh_toolkit/src/my_dh_toolkit/processor.py b/my_dh_toolkit/src/my_dh_toolkit/processor.py index 526c766..65da8b6 100644 --- a/my_dh_toolkit/src/my_dh_toolkit/processor.py +++ b/my_dh_toolkit/src/my_dh_toolkit/processor.py @@ -57,6 +57,10 @@ def batch_process(directory: str, output_dir: str, verbose: bool = False) -> Non @click.option("--verbose", "-v", is_flag=True, help="Enable verbose output") def process(directory: str, output: str, verbose: bool) -> None: """Batch process CSV files with Deephaven.""" + from deephaven_server import Server + + Server(port=10000, jvm_args=["-Xmx4g"]).start() + batch_process(directory, output, verbose) click.echo("Batch processing complete!") diff --git a/setuptools-deployment.md b/setuptools-deployment.md index 8af03a6..197e69e 100644 --- a/setuptools-deployment.md +++ b/setuptools-deployment.md @@ -55,6 +55,29 @@ my_dh_project/ The package name under `src/` determines how users import your code. For example, with `src/my_dh_library/`, users import via `from my_dh_library import ...`. +## Server initialization + +Deephaven requires a running server before using any Deephaven functionality. The server must be initialized in the same Python process that uses Deephaven: + +```python +from deephaven_server import Server + +# Initialize and start the server +server = Server(port=10000, jvm_args=["-Xmx4g"]) +server.start() + +# Now you can import and use Deephaven +from deephaven import read_csv +data = read_csv("data.csv") +``` + +### Key points + +- Each Python process has its own JVM +- Starting a server in one terminal doesn't help another terminal +- Entry-point CLI commands should start their own server internally (see [Use CLI functions](#use-cli-functions)) so they work standalone; only functions imported directly need an already-running session +- The server uses approximately 4GB of memory by default (configurable via `jvm_args`) + ## Packaging scenarios Different projects have different needs. The example repository demonstrates three common scenarios: @@ -111,10 +134,8 @@ my_dh_cli/ **Usage:** -```python -# CLI functions are used within a Python session -from my_dh_cli.cli import my_dh_query -result = my_dh_query("data.csv", verbose=True) +```bash +my-dh-query data.csv --verbose ``` **When to use:** @@ -148,14 +169,15 @@ my_dh_toolkit/ ```python # As a library from my_dh_toolkit.queries import filter_by_threshold -from my_dh_toolkit import my_dh_query -# Use library functions data = read_csv("data.csv") filtered = filter_by_threshold(data, "Score", 75.0) +``` -# Or use CLI functions -result = my_dh_query("data.csv", verbose=True) +```bash +# As CLI commands +my-dh-query data.csv --verbose +my-dh-process data/ --output results/ --verbose ``` **When to use:** @@ -396,6 +418,10 @@ def my_dh_query(input_file: str, verbose: bool = False): @click.option("--verbose", "-v", is_flag=True, help="Enable verbose output") def app(input_file: str, verbose: bool) -> None: """Process data with Deephaven.""" + from deephaven_server import Server + + Server(port=10000, jvm_args=["-Xmx4g"]).start() + result = my_dh_query(input_file, verbose) click.echo("Processing complete!") @@ -529,6 +555,10 @@ def batch_process(directory: str, output_dir: str, verbose: bool = False) -> Non @click.option("--verbose", "-v", is_flag=True, help="Enable verbose output") def process(directory: str, output: str, verbose: bool) -> None: """Batch process CSV files with Deephaven.""" + from deephaven_server import Server + + Server(port=10000, jvm_args=["-Xmx4g"]).start() + batch_process(directory, output, verbose) click.echo("Batch processing complete!") @@ -670,7 +700,13 @@ filtered = filter_by_threshold(data, "Score", 75.0) ### Use CLI functions -CLI functions must be used within the same Python session as the server: +Entry-point commands like `my-dh-query` start their own Deephaven server, so they run as standalone terminal commands: + +```bash +my-dh-query data.csv --verbose +``` + +The underlying function is also importable, so you can call it directly within a Python session that already has a server running: ```python # Start the Deephaven server @@ -678,7 +714,7 @@ from deephaven_server import Server server = Server(port=10000, jvm_args=["-Xmx4g"]) server.start() -# Use CLI functions +# Call the underlying function directly from my_dh_cli.cli import my_dh_query result = my_dh_query("data.csv", verbose=True) ``` @@ -729,29 +765,6 @@ This creates a `.whl` file in `dist/` that can be: - Include sample data for testing - Document how to run tests -## Server initialization - -Deephaven requires a running server before using any Deephaven functionality. The server must be initialized in the same Python process that uses Deephaven: - -```python -from deephaven_server import Server - -# Initialize and start the server -server = Server(port=10000, jvm_args=["-Xmx4g"]) -server.start() - -# Now you can import and use Deephaven -from deephaven import read_csv -data = read_csv("data.csv") -``` - -### Key points - -- Each Python process has its own JVM -- Starting a server in one terminal doesn't help another terminal -- CLI tools must run in the same session as the server -- The server uses approximately 4GB of memory by default (configurable via `jvm_args`) - ## Next steps The [deephaven-python-packaging](https://github.com/deephaven-examples/deephaven-python-packaging) repository provides complete, working examples of all three packaging scenarios. Clone the repository and explore the examples to see how to structure your own Deephaven packages. From 339df3843dc9b283fab061479bfc5e2b6805e74f Mon Sep 17 00:00:00 2001 From: elijahpetty Date: Tue, 1 Sep 2026 12:43:14 -0500 Subject: [PATCH 07/13] . --- setuptools-deployment.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/setuptools-deployment.md b/setuptools-deployment.md index 197e69e..b1c2768 100644 --- a/setuptools-deployment.md +++ b/setuptools-deployment.md @@ -73,10 +73,10 @@ data = read_csv("data.csv") ### Key points -- Each Python process has its own JVM -- Starting a server in one terminal doesn't help another terminal -- Entry-point CLI commands should start their own server internally (see [Use CLI functions](#use-cli-functions)) so they work standalone; only functions imported directly need an already-running session -- The server uses approximately 4GB of memory by default (configurable via `jvm_args`) +- Each Python process has its own JVM. +- Starting a server in one terminal doesn't help another terminal. +- Entry-point CLI commands should start their own server internally (see [Use CLI functions](#use-cli-functions)) so they work standalone; only functions imported directly need an already-running session. +- The server uses approximately 4GB of memory by default (configurable via `jvm_args`). ## Packaging scenarios @@ -109,10 +109,10 @@ data = read_csv("data.csv") filtered = filter_by_threshold(data, "Score", 75.0) ``` -**When to use:** +**Use when:** - Creating reusable utilities for other projects. -- No command-line interface needed. +- You don't need a command-line interface. - Code will be imported, not executed directly. ### CLI-only package @@ -138,11 +138,11 @@ my_dh_cli/ my-dh-query data.csv --verbose ``` -**When to use:** +**Use when:** - Building command-line tools for data processing -- Want clean function interfaces -- No library code to expose to other projects +- You need clean function interfaces +- You don't need to expose library code to other projects ### Combined package @@ -180,10 +180,10 @@ my-dh-query data.csv --verbose my-dh-process data/ --output results/ --verbose ``` -**When to use:** +**Use when:** -- Need both library and CLI functionality -- Want to provide multiple interfaces to the same code +- You need both library and CLI functionality +- You want to provide multiple interfaces to the same code - Library functions are useful independently ## Create a new package @@ -603,7 +603,7 @@ where = ["src"] - **`[build-system]`** - Specifies setuptools as the build backend - **`[project]`** - Package metadata and dependencies - **`name`** - Project name (used for `pip install`) -- **`dependencies`** - Required packages installed automatically +- **`dependencies`** - Required packages, installed automatically - **`[tool.setuptools.packages.find]`** - Tells setuptools to find packages in `src/` For CLI packages, add a `[project.scripts]` section: From f111a48ca723a8084804c762da6cc9541c2c6e58 Mon Sep 17 00:00:00 2001 From: elijahpetty Date: Wed, 9 Sep 2026 16:16:47 -0500 Subject: [PATCH 08/13] impl suggestions --- README.md | 14 +++++++------- my_dh_cli/README.md | 6 +++--- my_dh_cli/data/batch/file1.csv | 4 ---- my_dh_cli/data/batch/file2.csv | 4 ---- my_dh_cli/data/batch/file3.csv | 5 ----- my_dh_cli/data/sample.csv | 11 ----------- my_dh_cli/pyproject.toml | 2 +- my_dh_library/README.md | 4 ++-- my_dh_library/data/batch/file1.csv | 4 ---- my_dh_library/data/batch/file2.csv | 4 ---- my_dh_library/data/batch/file3.csv | 5 ----- my_dh_library/data/sample.csv | 11 ----------- my_dh_library/pyproject.toml | 2 +- my_dh_library/src/my_dh_library/utils.py | 2 ++ my_dh_toolkit/README.md | 20 ++++++++++---------- my_dh_toolkit/data/batch/file1.csv | 4 ---- my_dh_toolkit/data/batch/file2.csv | 4 ---- my_dh_toolkit/data/batch/file3.csv | 5 ----- my_dh_toolkit/data/sample.csv | 11 ----------- my_dh_toolkit/pyproject.toml | 6 +++--- my_dh_toolkit/src/my_dh_toolkit/utils.py | 2 ++ setuptools-deployment.md | 16 ++++++++-------- 22 files changed, 39 insertions(+), 107 deletions(-) delete mode 100644 my_dh_cli/data/batch/file1.csv delete mode 100644 my_dh_cli/data/batch/file2.csv delete mode 100644 my_dh_cli/data/batch/file3.csv delete mode 100644 my_dh_cli/data/sample.csv delete mode 100644 my_dh_library/data/batch/file1.csv delete mode 100644 my_dh_library/data/batch/file2.csv delete mode 100644 my_dh_library/data/batch/file3.csv delete mode 100644 my_dh_library/data/sample.csv delete mode 100644 my_dh_toolkit/data/batch/file1.csv delete mode 100644 my_dh_toolkit/data/batch/file2.csv delete mode 100644 my_dh_toolkit/data/batch/file3.csv delete mode 100644 my_dh_toolkit/data/sample.csv diff --git a/README.md b/README.md index 84ecf7b..03ee56d 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ This example accompanies the [Packaging custom code and dependencies](https://de ## Prerequisites -- Python 3.8 or later +- Python 3.9 or later - pip (Python package installer) - Basic familiarity with Python packaging @@ -89,8 +89,8 @@ from my_dh_toolkit.queries import filter_by_threshold ```shell # As CLI commands -my-dh-query input_data.csv --verbose -my-dh-process data/ --output results/ --verbose +my-dh-toolkit-query input_data.csv --verbose +my-dh-toolkit-process data/ --output results/ --verbose ``` ## Quick start @@ -152,13 +152,13 @@ print(f"Processed {result.size} rows") ### Try the combined package -Install the package, then run the installed commands directly from a terminal. Both `my-dh-query` and `my-dh-process` start their own Deephaven server: +Install the package, then run the installed commands directly from a terminal. Both `my-dh-toolkit-query` and `my-dh-toolkit-process` start their own Deephaven server: ```shell cd my_dh_toolkit pip install -e . -my-dh-query ../data/sample.csv --verbose -my-dh-process ../data/batch --output ./output --verbose +my-dh-toolkit-query ../data/sample.csv --verbose +my-dh-toolkit-process ../data/batch --output ./output --verbose ``` The library functions are also importable for use within a Python session that already has a server running: @@ -338,7 +338,7 @@ python -m my_package ### Import errors - Verify all dependencies are installed: `pip list` -- Check that you're using Python 3.8 or later +- Check that you're using Python 3.9 or later - Ensure Deephaven is installed: `pip install deephaven-server` ### Module not found errors diff --git a/my_dh_cli/README.md b/my_dh_cli/README.md index 709a274..d755507 100644 --- a/my_dh_cli/README.md +++ b/my_dh_cli/README.md @@ -19,7 +19,7 @@ pip install -e . Run the installed command directly from a terminal. `my-dh-query` starts its own Deephaven server, so no separate session setup is needed: ```shell -my-dh-query data/sample.csv --verbose +my-dh-query ../data/sample.csv --verbose ``` The underlying `my_dh_query()` function is also importable, so you can call it directly within a Python session that already has a server running: @@ -32,7 +32,7 @@ server.start() # Call the underlying function directly from my_dh_cli.cli import my_dh_query -result = my_dh_query("data/sample.csv", verbose=True) +result = my_dh_query("../data/sample.csv", verbose=True) print(f"Processed {result.size} rows") ``` @@ -50,6 +50,6 @@ Process a CSV file with Deephaven. ## Requirements -- Python 3.8 or later +- Python 3.9 or later - Deephaven Server 0.35.0 or later - Click 8.0.0 or later diff --git a/my_dh_cli/data/batch/file1.csv b/my_dh_cli/data/batch/file1.csv deleted file mode 100644 index fe9fae6..0000000 --- a/my_dh_cli/data/batch/file1.csv +++ /dev/null @@ -1,4 +0,0 @@ -Name,Score,Value,Category -Alice,85,120,A -Bob,92,150,B -Charlie,78,95,A diff --git a/my_dh_cli/data/batch/file2.csv b/my_dh_cli/data/batch/file2.csv deleted file mode 100644 index e270a59..0000000 --- a/my_dh_cli/data/batch/file2.csv +++ /dev/null @@ -1,4 +0,0 @@ -Name,Score,Value,Category -Diana,88,110,C -Eve,95,180,B -Frank,72,85,A diff --git a/my_dh_cli/data/batch/file3.csv b/my_dh_cli/data/batch/file3.csv deleted file mode 100644 index 692c4a2..0000000 --- a/my_dh_cli/data/batch/file3.csv +++ /dev/null @@ -1,5 +0,0 @@ -Name,Score,Value,Category -Grace,91,160,C -Henry,83,105,B -Iris,89,140,A -Jack,76,90,C diff --git a/my_dh_cli/data/sample.csv b/my_dh_cli/data/sample.csv deleted file mode 100644 index 1d1b856..0000000 --- a/my_dh_cli/data/sample.csv +++ /dev/null @@ -1,11 +0,0 @@ -Name,Score,Value,Category -Alice,85,120,A -Bob,92,150,B -Charlie,78,95,A -Diana,88,110,C -Eve,95,180,B -Frank,72,85,A -Grace,91,160,C -Henry,83,105,B -Iris,89,140,A -Jack,76,90,C diff --git a/my_dh_cli/pyproject.toml b/my_dh_cli/pyproject.toml index 32c7f84..6e844c5 100644 --- a/my_dh_cli/pyproject.toml +++ b/my_dh_cli/pyproject.toml @@ -7,7 +7,7 @@ name = "my_dh_cli" version = "0.1.0" description = "Command-line tool for data processing" readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.9" dependencies = [ "deephaven-server>=0.35.0", "click>=8.0.0", diff --git a/my_dh_library/README.md b/my_dh_library/README.md index 4362a1c..887817f 100644 --- a/my_dh_library/README.md +++ b/my_dh_library/README.md @@ -31,7 +31,7 @@ server.start() from my_dh_library.queries import filter_by_threshold, add_computed_columns from deephaven import read_csv -data = read_csv("data/sample.csv") +data = read_csv("../data/sample.csv") filtered = filter_by_threshold(data, "Score", 75.0) enhanced = add_computed_columns(filtered) ``` @@ -51,5 +51,5 @@ enhanced = add_computed_columns(filtered) ## Requirements -- Python 3.8 or later +- Python 3.9 or later - Deephaven Server 0.35.0 or later diff --git a/my_dh_library/data/batch/file1.csv b/my_dh_library/data/batch/file1.csv deleted file mode 100644 index fe9fae6..0000000 --- a/my_dh_library/data/batch/file1.csv +++ /dev/null @@ -1,4 +0,0 @@ -Name,Score,Value,Category -Alice,85,120,A -Bob,92,150,B -Charlie,78,95,A diff --git a/my_dh_library/data/batch/file2.csv b/my_dh_library/data/batch/file2.csv deleted file mode 100644 index e270a59..0000000 --- a/my_dh_library/data/batch/file2.csv +++ /dev/null @@ -1,4 +0,0 @@ -Name,Score,Value,Category -Diana,88,110,C -Eve,95,180,B -Frank,72,85,A diff --git a/my_dh_library/data/batch/file3.csv b/my_dh_library/data/batch/file3.csv deleted file mode 100644 index 692c4a2..0000000 --- a/my_dh_library/data/batch/file3.csv +++ /dev/null @@ -1,5 +0,0 @@ -Name,Score,Value,Category -Grace,91,160,C -Henry,83,105,B -Iris,89,140,A -Jack,76,90,C diff --git a/my_dh_library/data/sample.csv b/my_dh_library/data/sample.csv deleted file mode 100644 index 1d1b856..0000000 --- a/my_dh_library/data/sample.csv +++ /dev/null @@ -1,11 +0,0 @@ -Name,Score,Value,Category -Alice,85,120,A -Bob,92,150,B -Charlie,78,95,A -Diana,88,110,C -Eve,95,180,B -Frank,72,85,A -Grace,91,160,C -Henry,83,105,B -Iris,89,140,A -Jack,76,90,C diff --git a/my_dh_library/pyproject.toml b/my_dh_library/pyproject.toml index 3fa6b90..aee4a0b 100644 --- a/my_dh_library/pyproject.toml +++ b/my_dh_library/pyproject.toml @@ -7,7 +7,7 @@ name = "my_dh_library" version = "0.1.0" description = "Reusable Deephaven query functions" readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.9" dependencies = [ "deephaven-server>=0.35.0", ] diff --git a/my_dh_library/src/my_dh_library/utils.py b/my_dh_library/src/my_dh_library/utils.py index f38c859..c4d1abb 100644 --- a/my_dh_library/src/my_dh_library/utils.py +++ b/my_dh_library/src/my_dh_library/utils.py @@ -1,5 +1,7 @@ """Utility functions for working with Deephaven tables.""" +from __future__ import annotations + from deephaven.table import Table diff --git a/my_dh_toolkit/README.md b/my_dh_toolkit/README.md index 6bddf5f..244e04f 100644 --- a/my_dh_toolkit/README.md +++ b/my_dh_toolkit/README.md @@ -32,20 +32,20 @@ from my_dh_toolkit.queries import filter_by_threshold, add_computed_columns from my_dh_toolkit import my_dh_query, batch_process from deephaven import read_csv -data = read_csv("data/sample.csv") +data = read_csv("../data/sample.csv") filtered = filter_by_threshold(data, "Score", 75.0) # Or use the exported functions -result = my_dh_query("data/sample.csv", verbose=True) +result = my_dh_query("../data/sample.csv", verbose=True) ``` ## Usage as CLI Commands -Run the installed commands directly from a terminal. Both `my-dh-query` and `my-dh-process` start their own Deephaven server: +Run the installed commands directly from a terminal. Both `my-dh-toolkit-query` and `my-dh-toolkit-process` start their own Deephaven server: ```shell -my-dh-query data/sample.csv --verbose -my-dh-process data/batch --output ./output --verbose +my-dh-toolkit-query ../data/sample.csv --verbose +my-dh-toolkit-process ../data/batch --output ./output --verbose ``` The underlying functions are also importable, so you can call them directly within a Python session that already has a server running: @@ -58,13 +58,13 @@ server.start() # Call the underlying functions directly from my_dh_toolkit import my_dh_query, batch_process -result = my_dh_query("data/sample.csv", verbose=True) -batch_process("data/batch/", "./output", verbose=True) +result = my_dh_query("../data/sample.csv", verbose=True) +batch_process("../data/batch/", "./output", verbose=True) ``` ## Commands -### my-dh-query +### my-dh-toolkit-query Process a single CSV file with Deephaven. @@ -74,7 +74,7 @@ Process a single CSV file with Deephaven. **Options:** - `--verbose, -v` - Enable verbose output -### my-dh-process +### my-dh-toolkit-process Batch process multiple CSV files from a directory. @@ -105,6 +105,6 @@ Batch process multiple CSV files from a directory. ## Requirements -- Python 3.8 or later +- Python 3.9 or later - Deephaven Server 0.35.0 or later - Click 8.0.0 or later diff --git a/my_dh_toolkit/data/batch/file1.csv b/my_dh_toolkit/data/batch/file1.csv deleted file mode 100644 index fe9fae6..0000000 --- a/my_dh_toolkit/data/batch/file1.csv +++ /dev/null @@ -1,4 +0,0 @@ -Name,Score,Value,Category -Alice,85,120,A -Bob,92,150,B -Charlie,78,95,A diff --git a/my_dh_toolkit/data/batch/file2.csv b/my_dh_toolkit/data/batch/file2.csv deleted file mode 100644 index e270a59..0000000 --- a/my_dh_toolkit/data/batch/file2.csv +++ /dev/null @@ -1,4 +0,0 @@ -Name,Score,Value,Category -Diana,88,110,C -Eve,95,180,B -Frank,72,85,A diff --git a/my_dh_toolkit/data/batch/file3.csv b/my_dh_toolkit/data/batch/file3.csv deleted file mode 100644 index 692c4a2..0000000 --- a/my_dh_toolkit/data/batch/file3.csv +++ /dev/null @@ -1,5 +0,0 @@ -Name,Score,Value,Category -Grace,91,160,C -Henry,83,105,B -Iris,89,140,A -Jack,76,90,C diff --git a/my_dh_toolkit/data/sample.csv b/my_dh_toolkit/data/sample.csv deleted file mode 100644 index 1d1b856..0000000 --- a/my_dh_toolkit/data/sample.csv +++ /dev/null @@ -1,11 +0,0 @@ -Name,Score,Value,Category -Alice,85,120,A -Bob,92,150,B -Charlie,78,95,A -Diana,88,110,C -Eve,95,180,B -Frank,72,85,A -Grace,91,160,C -Henry,83,105,B -Iris,89,140,A -Jack,76,90,C diff --git a/my_dh_toolkit/pyproject.toml b/my_dh_toolkit/pyproject.toml index 81bc624..1051e34 100644 --- a/my_dh_toolkit/pyproject.toml +++ b/my_dh_toolkit/pyproject.toml @@ -7,15 +7,15 @@ name = "my_dh_toolkit" version = "0.1.0" description = "Deephaven library and CLI tools" readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.9" dependencies = [ "deephaven-server>=0.35.0", "click>=8.0.0", ] [project.scripts] -my-dh-query = "my_dh_toolkit.cli:app" -my-dh-process = "my_dh_toolkit.processor:process" +my-dh-toolkit-query = "my_dh_toolkit.cli:app" +my-dh-toolkit-process = "my_dh_toolkit.processor:process" [tool.setuptools.packages.find] where = ["src"] diff --git a/my_dh_toolkit/src/my_dh_toolkit/utils.py b/my_dh_toolkit/src/my_dh_toolkit/utils.py index f38c859..c4d1abb 100644 --- a/my_dh_toolkit/src/my_dh_toolkit/utils.py +++ b/my_dh_toolkit/src/my_dh_toolkit/utils.py @@ -1,5 +1,7 @@ """Utility functions for working with Deephaven tables.""" +from __future__ import annotations + from deephaven.table import Table diff --git a/setuptools-deployment.md b/setuptools-deployment.md index b1c2768..e5fea0c 100644 --- a/setuptools-deployment.md +++ b/setuptools-deployment.md @@ -176,8 +176,8 @@ filtered = filter_by_threshold(data, "Score", 75.0) ```bash # As CLI commands -my-dh-query data.csv --verbose -my-dh-process data/ --output results/ --verbose +my-dh-toolkit-query data.csv --verbose +my-dh-toolkit-process data/ --output results/ --verbose ``` **Use when:** @@ -214,7 +214,7 @@ name = "my_dh_library" version = "0.1.0" description = "Reusable Deephaven query functions" readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.9" dependencies = [ "deephaven-server>=0.35.0", ] @@ -342,7 +342,7 @@ name = "my_dh_cli" version = "0.1.0" description = "Command-line tool for data processing" readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.9" dependencies = [ "deephaven-server>=0.35.0", "click>=8.0.0", @@ -453,15 +453,15 @@ name = "my_dh_toolkit" version = "0.1.0" description = "Deephaven library and CLI tools" readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.9" dependencies = [ "deephaven-server>=0.35.0", "click>=8.0.0", ] [project.scripts] -my-dh-query = "my_dh_toolkit.cli:app" -my-dh-process = "my_dh_toolkit.processor:process" +my-dh-toolkit-query = "my_dh_toolkit.cli:app" +my-dh-toolkit-process = "my_dh_toolkit.processor:process" [tool.setuptools.packages.find] where = ["src"] @@ -589,7 +589,7 @@ name = "my_dh_library" version = "0.1.0" description = "Reusable Deephaven query functions" readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.9" dependencies = [ "deephaven-server>=0.35.0", ] From bc71cf1b18e8ae23e944faf9ecbc3d21686d6b6c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:32:13 +0000 Subject: [PATCH 09/13] Give each example a single clear story and make READMEs usable end-to-end Co-authored-by: chipkent <5250374+chipkent@users.noreply.github.com> --- .gitignore | 1 + README.md | 394 +++++++------------ data/batch/file1.csv | 6 +- data/batch/file2.csv | 6 +- data/batch/file3.csv | 8 +- my_dh_cli/README.md | 44 ++- my_dh_library/README.md | 38 +- my_dh_toolkit/README.md | 113 +++--- my_dh_toolkit/src/my_dh_toolkit/__init__.py | 13 +- my_dh_toolkit/src/my_dh_toolkit/processor.py | 5 +- setuptools-deployment.md | 34 +- 11 files changed, 280 insertions(+), 382 deletions(-) diff --git a/.gitignore b/.gitignore index f07e50c..6c540cd 100644 --- a/.gitignore +++ b/.gitignore @@ -40,4 +40,5 @@ Thumbs.db # Test results results/ +output/ *.log \ No newline at end of file diff --git a/README.md b/README.md index 03ee56d..cdee4c8 100644 --- a/README.md +++ b/README.md @@ -1,355 +1,249 @@ -# Python Packaging with Deephaven +# Deephaven Python packaging examples -This repository demonstrates how to create and deploy Python packages that use Deephaven Community Core. It shows three complete packaging scenarios following the official [Python Packaging User Guide](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/) recommendations. +This repository shows how to package Python code that uses [Deephaven Community Core](https://deephaven.io/community/) so that it can be installed with `pip`. It contains three small, self-contained example packages. Each example demonstrates exactly one packaging pattern: -This example accompanies the [Packaging custom code and dependencies](https://deephaven.io/core/docs/how-to-guides/sysadmin/setuptools-deployment/) guide in the Deephaven documentation. +| Example | Pattern | Installing it provides | +|---|---|---| +| [`my_dh_library/`](my_dh_library/) | Library only | Functions to import in Python code | +| [`my_dh_cli/`](my_dh_cli/) | Command-line tool only | A `my-dh-query` terminal command | +| [`my_dh_toolkit/`](my_dh_toolkit/) | Library and command-line tools combined | Importable functions plus `my-dh-toolkit-query` and `my-dh-toolkit-process` commands | -## What you'll learn +`my_dh_toolkit` is the other two patterns merged into a single package: its library modules play the same role as `my_dh_library`, and its commands play the same role as `my_dh_cli`. -- Create installable Python packages with Deephaven dependencies -- Package reusable library code for other projects -- Build command-line tools with entry point scripts -- Manage dependencies with `pyproject.toml` -- Use the src-layout structure -- Distribute packages as wheel archives +All three examples follow the [Python Packaging User Guide](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/) conventions: a `pyproject.toml` file for metadata, dependencies, and entry points, and the src-layout for source code. This repository accompanies the [Packaging custom code and dependencies](https://deephaven.io/core/docs/how-to-guides/sysadmin/setuptools-deployment/) guide, which explains the underlying concepts in depth. -## Prerequisites - -- Python 3.9 or later -- pip (Python package installer) -- Basic familiarity with Python packaging - -## Repository structure - -This repository contains three complete packaging scenarios: +## Choose an example -### 1. Library-only package (`my_dh_library/`) - -Reusable library code without CLI tools. Other projects import your modules. - -``` -my_dh_library/ -├── src/ -│ └── my_dh_library/ -│ ├── __init__.py -│ ├── queries.py -│ └── utils.py -├── pyproject.toml -└── README.md -``` +- Start from **`my_dh_library`** to share reusable functions that other projects import. There is no command-line interface. +- Start from **`my_dh_cli`** to ship a tool that users run from a terminal. No library code is exposed. +- Start from **`my_dh_toolkit`** to provide both: importable functions for Python users and commands for terminal users. -**Usage:** -```python -from my_dh_library.queries import filter_by_threshold -``` +## Prerequisites -### 2. CLI-only package (`my_dh_cli/`) +- Python 3.9 or later. +- pip. +- Java 17 or later (required by `deephaven-server`, which each example installs as a dependency). -Command-line tool without exposing library code. +## Get the examples -``` -my_dh_cli/ -├── src/ -│ └── my_dh_cli/ -│ ├── __init__.py -│ ├── __main__.py -│ └── cli.py -├── pyproject.toml -└── README.md -``` +Clone the repository and work from its root directory. All commands below are run from the repository root. -**Usage:** ```shell -my-dh-query input_data.csv --verbose +git clone https://github.com/deephaven-examples/deephaven-python-packaging.git +cd deephaven-python-packaging ``` -### 3. Combined package (`my_dh_toolkit/`) +## Example 1: `my_dh_library` — a library -Both reusable library code and command-line tools. +**The story:** package reusable Deephaven query functions so that other projects can `pip install` the package and import the functions. ``` -my_dh_toolkit/ +my_dh_library/ ├── src/ -│ └── my_dh_toolkit/ -│ ├── __init__.py -│ ├── __main__.py -│ ├── cli.py -│ ├── processor.py -│ ├── queries.py -│ └── utils.py -├── pyproject.toml +│ └── my_dh_library/ +│ ├── __init__.py # Exports the public API +│ ├── queries.py # Query functions: filter, compute, summarize +│ └── utils.py # Table validation helpers +├── pyproject.toml # Declares metadata and the deephaven-server dependency └── README.md ``` -**Usage:** -```python -# As a library -from my_dh_toolkit.queries import filter_by_threshold -``` - -```shell -# As CLI commands -my-dh-toolkit-query input_data.csv --verbose -my-dh-toolkit-process data/ --output results/ --verbose -``` - -## Quick start +There is no `[project.scripts]` section in `pyproject.toml` and no `__main__.py` — this package is only ever imported. -Clone the repository: +### Try it -```shell -git clone https://github.com/deephaven-examples/deephaven-python-packaging.git -cd deephaven-python-packaging -``` - -### Try the library-only package +Install the package and start Python: ```shell -cd my_dh_library -pip install -e . +pip install -e ./my_dh_library python ``` -Then in Python: +A library that uses Deephaven needs a running server in the same process, so start one before importing `deephaven` modules: ```python -# Start the Deephaven server +# A Deephaven server must be running before deephaven modules are imported. from deephaven_server import Server -server = Server(port=10000, jvm_args=["-Xmx4g"]) -server.start() +Server(port=10000, jvm_args=["-Xmx4g"]).start() -# Now use the library functions +# Import and use the installed library. from my_dh_library.queries import filter_by_threshold from deephaven import read_csv -data = read_csv("../data/sample.csv") +data = read_csv("data/sample.csv") filtered = filter_by_threshold(data, "Score", 75.0) +print(f"{filtered.size} of {data.size} rows have Score > 75") ``` -### Try the CLI-only package +### What to study -Install the package, then run the installed command directly from a terminal. `my-dh-query` starts its own Deephaven server, so no separate session setup is needed: +- [`pyproject.toml`](my_dh_library/pyproject.toml) — the `dependencies` list installs `deephaven-server` automatically, and `[tool.setuptools.packages.find]` points setuptools at `src/`. +- [`queries.py`](my_dh_library/src/my_dh_library/queries.py) — plain functions that take and return Deephaven tables. +- [`__init__.py`](my_dh_library/src/my_dh_library/__init__.py) — re-exports the public functions. -```shell -cd my_dh_cli -pip install -e . -my-dh-query ../data/sample.csv --verbose -``` +## Example 2: `my_dh_cli` — a command-line tool -The underlying `my_dh_query()` function is also importable, so you can call it directly within a Python session that already has a server running (useful when composing it with other Deephaven code): +**The story:** package a Deephaven script as a terminal command. `pip install` creates a `my-dh-query` command that users run without writing any Python. -```python -# Start the Deephaven server -from deephaven_server import Server -server = Server(port=10000, jvm_args=["-Xmx4g"]) -server.start() - -# Call the underlying function directly -from my_dh_cli.cli import my_dh_query -result = my_dh_query("../data/sample.csv", verbose=True) -print(f"Processed {result.size} rows") ``` - -### Try the combined package - -Install the package, then run the installed commands directly from a terminal. Both `my-dh-toolkit-query` and `my-dh-toolkit-process` start their own Deephaven server: - -```shell -cd my_dh_toolkit -pip install -e . -my-dh-toolkit-query ../data/sample.csv --verbose -my-dh-toolkit-process ../data/batch --output ./output --verbose +my_dh_cli/ +├── src/ +│ └── my_dh_cli/ +│ ├── __init__.py +│ ├── __main__.py # Enables `python -m my_dh_cli` during development +│ └── cli.py # The command implementation +├── pyproject.toml # Declares the my-dh-query entry point +└── README.md ``` -The library functions are also importable for use within a Python session that already has a server running: +The command comes from one line in `pyproject.toml`: -```python -# Start the Deephaven server -from deephaven_server import Server -server = Server(port=10000, jvm_args=["-Xmx4g"]) -server.start() +```toml +[project.scripts] +my-dh-query = "my_dh_cli.cli:app" +``` -# Use as a library -from my_dh_toolkit.queries import filter_by_threshold -from deephaven import read_csv +### Try it -data = read_csv("../data/sample.csv") -filtered = filter_by_threshold(data, "Score", 75.0) +Install the package, then run the command on the sample data: -# Or call the underlying CLI functions directly -from my_dh_toolkit import my_dh_query, batch_process -result = my_dh_query("../data/sample.csv", verbose=True) -batch_process("../data/batch/", "./output", verbose=True) +```shell +pip install -e ./my_dh_cli +my-dh-query data/sample.csv --verbose ``` -## Sample data - -The `data/` directory contains sample CSV files for testing: +The command starts its own Deephaven server, reads the CSV file, adds a computed column, and reports the row count. No separate setup is needed. -- `sample.csv` - Single file with Name, Score, Value, and Category columns -- `batch/` - Multiple CSV files for batch processing examples +### What to study -## Key concepts +- [`pyproject.toml`](my_dh_cli/pyproject.toml) — the `[project.scripts]` section maps the command name to a function. +- [`cli.py`](my_dh_cli/src/my_dh_cli/cli.py) — a [Click](https://click.palletsprojects.com/) command that starts the Deephaven server itself, so it works as a standalone tool. +- [`__main__.py`](my_dh_cli/src/my_dh_cli/__main__.py) — allows `python -m my_dh_cli data/sample.csv` as an alternative during development. -### Package structure +## Example 3: `my_dh_toolkit` — a library and commands in one package -All examples use the **src-layout**, which is the recommended structure for Python packages: +**The story:** one package, two interfaces. Terminal users get commands; Python users import functions. The library modules (`queries.py`, `utils.py`) match `my_dh_library`, and the command modules (`cli.py`, `processor.py`) follow the same pattern as `my_dh_cli`. ``` -my_project/ +my_dh_toolkit/ ├── src/ -│ └── my_package/ -│ ├── __init__.py -│ └── module.py -├── pyproject.toml +│ └── my_dh_toolkit/ +│ ├── __init__.py # Kept minimal — see "What to study" below +│ ├── __main__.py +│ ├── cli.py # my-dh-toolkit-query command +│ ├── processor.py # my-dh-toolkit-process command +│ ├── queries.py # Library: query functions +│ └── utils.py # Library: table validation helpers +├── pyproject.toml # Declares two entry points └── README.md ``` -The src-layout keeps source code separate from tests and configuration files. +### Try the commands -### Entry point scripts +Install the package, then run the two commands: -Entry point scripts are defined in `[project.scripts]` and become available after installation: - -```toml -[project.scripts] -my-command = "my_package.module:function" +```shell +pip install -e ./my_dh_toolkit +my-dh-toolkit-query data/sample.csv --verbose +my-dh-toolkit-process data/batch --output output --verbose ``` -After `pip install`, you can run `my-command` from anywhere. +`my-dh-toolkit-query` processes a single CSV file. `my-dh-toolkit-process` processes every CSV file in a directory and writes the results to the output directory. Both start their own Deephaven server. -### Module execution +### Try the library -Add a `__main__.py` file to support running packages with `python -m`: +The same installed package is importable. As with any Deephaven library, start a server first: ```python -from my_package.cli import app - -if __name__ == "__main__": - app() -``` - -This allows running without installation: `python -m my_package` - -### Dependencies +# A Deephaven server must be running before deephaven modules are imported. +from deephaven_server import Server +Server(port=10000, jvm_args=["-Xmx4g"]).start() -Dependencies are specified in `pyproject.toml`: +# Import and use the installed library. +from my_dh_toolkit.queries import filter_by_threshold +from deephaven import read_csv -```toml -[project] -dependencies = [ - "deephaven-server>=0.35.0", - "click>=8.0.0", -] +data = read_csv("data/sample.csv") +filtered = filter_by_threshold(data, "Score", 75.0) +print(f"{filtered.size} of {data.size} rows have Score > 75") ``` -These are automatically installed when users install your package. +### What to study -## Building and distributing +- [`pyproject.toml`](my_dh_toolkit/pyproject.toml) — `[project.scripts]` defines multiple commands for one package. +- [`__init__.py`](my_dh_toolkit/src/my_dh_toolkit/__init__.py) — deliberately imports nothing that requires Deephaven. The entry-point commands import the package before a server is running, so the library API stays in the `queries` and `utils` submodules, and the command modules defer their `deephaven` imports until after the server starts. This is the key structural difference from a library-only package. -Build a distributable wheel: - -```shell -cd my_dh_cli # or any package directory -pip install build -python -m build -``` - -This creates a `.whl` file in `dist/` that can be: +## Sample data -- Installed locally: `pip install dist/my_dh_cli-0.1.0-py3-none-any.whl` -- Distributed to others -- Published to PyPI: `python -m twine upload dist/*` +The `data/` directory holds the inputs used by the examples above: -## Packaging scenarios +- `data/sample.csv` — one 10-row file with `Name`, `Score`, `Value`, and `Category` columns. Input for the library snippets, `my-dh-query`, and `my-dh-toolkit-query`. +- `data/batch/file1.csv`, `file2.csv`, `file3.csv` — three separate files with the same columns but different rows. Input for `my-dh-toolkit-process`, which processes every CSV file in the directory. -### When to use library-only +## Adapt an example for your own project -- Creating reusable code for other projects -- No command-line interface needed -- Code will be imported, not executed +Each example is a template. To turn one into your own package: -**Example:** Data processing utilities, query functions, helper classes +1. **Copy the example** that matches your scenario: -### When to use CLI-only + ```shell + cp -r my_dh_cli my_tool + cd my_tool + ``` -- Building command-line tools for end users -- No library code to expose -- Want clean command names +2. **Rename the import package** — the directory under `src/` is the name used in `import` statements: -**Example:** Data conversion tools, file processors, automation scripts + ```shell + mv src/my_dh_cli src/my_tool + ``` -### When to use combined +3. **Update `pyproject.toml`** — set your own `name`, `version`, and `description`, and point any `[project.scripts]` entries at the new package: -- Need both library and CLI functionality -- Want to provide multiple interfaces -- Library functions useful on their own + ```toml + [project] + name = "my_tool" -**Example:** Data analysis toolkit with both API and CLI + [project.scripts] + my-tool = "my_tool.cli:app" + ``` -## Execution patterns +4. **Update internal imports** to the new package name (for example, `from my_tool.cli import app` in `__main__.py`). -### Entry point scripts (recommended for CLI tools) +5. **Replace the example logic** with your own code, and add any packages it needs to `dependencies` in `pyproject.toml`. Keep `deephaven-server` in the list so it installs automatically. -**Configure in `pyproject.toml`:** -```toml -[project.scripts] -my-command = "my_package.module:function" -``` +6. **Reinstall and test:** -**Run after installation:** -```shell -my-command -``` + ```shell + pip install -e . + my-tool --help + ``` -**Benefits:** -- Clean command names -- Available system-wide -- Standard Python packaging approach +Three names must stay in sync: the package directory under `src/`, the module paths in `[project.scripts]`, and the package name in `import` statements. -### Module execution (useful for development) +## Install and distribute -**Add `__main__.py`:** -```python -from my_package.cli import app +The examples above use editable installs (`pip install -e ./my_dh_cli`), which pick up source edits without reinstalling — ideal while developing. The other common options: -if __name__ == "__main__": - app() -``` +- **Regular install from source:** `pip install ./my_dh_cli` +- **Build and install a wheel** — the format to use when distributing a package to other machines or publishing to a package index: -**Run without installation:** -```shell -python -m my_package -``` + ```shell + pip install build + python -m build my_dh_cli + pip install my_dh_cli/dist/my_dh_cli-0.1.0-py3-none-any.whl + ``` -**Benefits:** -- No installation required -- Useful for development and testing -- Works from source directory + Wheels can be shared directly or published to PyPI with [`twine`](https://twine.readthedocs.io/). ## Troubleshooting -### Command not found after installation - -- Ensure installation completed without errors -- Check that the installation directory is in your PATH -- Try reinstalling: `pip install --force-reinstall .` - -### Import errors - -- Verify all dependencies are installed: `pip list` -- Check that you're using Python 3.9 or later -- Ensure Deephaven is installed: `pip install deephaven-server` - -### Module not found errors - -- Verify `__init__.py` files exist in all package directories -- Check that package names in `[project.scripts]` match your directory structure -- Try reinstalling in editable mode: `pip install -e .` +- **Command not found after installation** — confirm the install succeeded (`pip show my_dh_cli`) and that the Python scripts directory is on `PATH`. Installing inside an activated virtual environment avoids most `PATH` issues. +- **`deephaven` import errors** — the Deephaven server must be started (as shown in the library examples) before `deephaven` modules are imported, and Java 17 or later must be available. +- **Module not found after renaming** — check that the directory under `src/`, the `[project.scripts]` module paths, and the `import` statements all use the new package name, then reinstall with `pip install -e .`. ## Related documentation -- [Packaging custom code and dependencies](https://deephaven.io/core/docs/how-to-guides/sysadmin/setuptools-deployment/) - Complete guide +- [Packaging custom code and dependencies](https://deephaven.io/core/docs/how-to-guides/sysadmin/setuptools-deployment/) — the guide this repository accompanies. - [Install and use Python packages](https://deephaven.io/core/docs/how-to-guides/install-and-use-python-packages/) - [Use the Deephaven Python package](https://deephaven.io/core/docs/how-to-guides/deephaven-python-package/) - [Python Packaging User Guide](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/) diff --git a/data/batch/file1.csv b/data/batch/file1.csv index fe9fae6..954231a 100644 --- a/data/batch/file1.csv +++ b/data/batch/file1.csv @@ -1,4 +1,4 @@ Name,Score,Value,Category -Alice,85,120,A -Bob,92,150,B -Charlie,78,95,A +Kara,81,130,A +Liam,94,175,B +Mona,77,100,A diff --git a/data/batch/file2.csv b/data/batch/file2.csv index e270a59..6975da9 100644 --- a/data/batch/file2.csv +++ b/data/batch/file2.csv @@ -1,4 +1,4 @@ Name,Score,Value,Category -Diana,88,110,C -Eve,95,180,B -Frank,72,85,A +Nina,86,115,C +Omar,90,155,B +Pria,74,80,A diff --git a/data/batch/file3.csv b/data/batch/file3.csv index 692c4a2..733a416 100644 --- a/data/batch/file3.csv +++ b/data/batch/file3.csv @@ -1,5 +1,5 @@ Name,Score,Value,Category -Grace,91,160,C -Henry,83,105,B -Iris,89,140,A -Jack,76,90,C +Quinn,93,165,C +Rosa,84,125,B +Sam,79,95,A +Tara,88,145,C diff --git a/my_dh_cli/README.md b/my_dh_cli/README.md index d755507..80ae851 100644 --- a/my_dh_cli/README.md +++ b/my_dh_cli/README.md @@ -1,55 +1,61 @@ # My Deephaven CLI -A CLI-only package providing command-line tools for data processing with Deephaven. This package is designed to be installed and run as a command-line tool. +An example of packaging a Deephaven script as a command-line tool. Installing this package creates one terminal command, `my-dh-query`. No library code is exposed — users of this package never write Python. + +The command is defined by the `[project.scripts]` entry point in [`pyproject.toml`](pyproject.toml): + +```toml +[project.scripts] +my-dh-query = "my_dh_cli.cli:app" +``` ## Installation +From the repository root: + ```shell -pip install . +pip install ./my_dh_cli ``` Or in editable mode for development: ```shell -pip install -e . +pip install -e ./my_dh_cli ``` ## Usage -Run the installed command directly from a terminal. `my-dh-query` starts its own Deephaven server, so no separate session setup is needed: +Run the installed command on a CSV file. The command starts its own Deephaven server, so no separate setup is needed: ```shell -my-dh-query ../data/sample.csv --verbose +my-dh-query data/sample.csv --verbose ``` -The underlying `my_dh_query()` function is also importable, so you can call it directly within a Python session that already has a server running: +It reads the file, adds a `DoubleScore` computed column, and reports the number of rows processed. -```python -# Start the Deephaven server -from deephaven_server import Server -server = Server(port=10000, jvm_args=["-Xmx4g"]) -server.start() +During development, the package also runs without an entry point via [`__main__.py`](src/my_dh_cli/__main__.py): -# Call the underlying function directly -from my_dh_cli.cli import my_dh_query -result = my_dh_query("../data/sample.csv", verbose=True) -print(f"Processed {result.size} rows") +```shell +python -m my_dh_cli data/sample.csv --verbose ``` -## Commands +## Command reference ### my-dh-query -Process a CSV file with Deephaven. +Process a CSV file with Deephaven. The file must contain a `Score` column. **Arguments:** -- `input_file` - Path to the CSV file to process + +- `input_file` - Path to the CSV file to process. **Options:** -- `--verbose, -v` - Enable verbose output + +- `--verbose, -v` - Enable verbose output. ## Requirements - Python 3.9 or later +- Java 17 or later - Deephaven Server 0.35.0 or later - Click 8.0.0 or later diff --git a/my_dh_library/README.md b/my_dh_library/README.md index 887817f..86de540 100644 --- a/my_dh_library/README.md +++ b/my_dh_library/README.md @@ -1,55 +1,57 @@ # My Deephaven Library -A library-only package providing reusable Deephaven query functions. This package contains no CLI tools - it's designed to be imported and used as a library in other Python projects. +An example of packaging reusable Deephaven query functions as a library. Installing this package makes its functions importable from any Python code. There are no command-line tools — this package is only ever imported. ## Installation +From the repository root: + ```shell -pip install . +pip install ./my_dh_library ``` Or in editable mode for development: ```shell -pip install -e . +pip install -e ./my_dh_library ``` ## Usage > [!NOTE] -> All Deephaven functionality requires a running server. Start the server before importing Deephaven modules. +> All Deephaven functionality requires a running server in the same Python process. Start the server before importing `deephaven` modules. -Import and use the library functions in your Python code: +From the repository root, start Python and use the library: ```python -# Start the Deephaven server +# A Deephaven server must be running before deephaven modules are imported. from deephaven_server import Server -server = Server(port=10000, jvm_args=["-Xmx4g"]) -server.start() +Server(port=10000, jvm_args=["-Xmx4g"]).start() -# Now use the library functions +# Import and use the installed library. from my_dh_library.queries import filter_by_threshold, add_computed_columns from deephaven import read_csv -data = read_csv("../data/sample.csv") +data = read_csv("data/sample.csv") filtered = filter_by_threshold(data, "Score", 75.0) enhanced = add_computed_columns(filtered) ``` -## Available Functions +## Available functions -### Query Functions (`my_dh_library.queries`) +### Query functions (`my_dh_library.queries`) -- `filter_by_threshold(table, column, threshold)` - Filter table rows where column value exceeds threshold -- `add_computed_columns(table)` - Add commonly used computed columns to a table -- `summarize_by_group(table, group_col, value_col)` - Create summary statistics grouped by a column +- `filter_by_threshold(table, column, threshold)` - Filter table rows where the column value exceeds the threshold. +- `add_computed_columns(table)` - Add commonly used computed columns to a table. +- `summarize_by_group(table, group_col, value_col)` - Create summary statistics grouped by a column. -### Utility Functions (`my_dh_library.utils`) +### Utility functions (`my_dh_library.utils`) -- `validate_columns(table, required_columns)` - Check if table has all required columns -- `get_table_info(table)` - Get basic information about a table +- `validate_columns(table, required_columns)` - Check if a table has all required columns. +- `get_table_info(table)` - Get basic information about a table. ## Requirements - Python 3.9 or later +- Java 17 or later - Deephaven Server 0.35.0 or later diff --git a/my_dh_toolkit/README.md b/my_dh_toolkit/README.md index 244e04f..0f785cf 100644 --- a/my_dh_toolkit/README.md +++ b/my_dh_toolkit/README.md @@ -1,110 +1,107 @@ # My Deephaven Toolkit -A combined package providing both reusable library code and command-line tools for Deephaven. This package can be used as both a library (imported in Python code) and as standalone CLI commands. +An example of one package with two interfaces: + +- **A library** — importable query functions, matching the [`my_dh_library`](../my_dh_library/) example. +- **Command-line tools** — two terminal commands, following the same pattern as the [`my_dh_cli`](../my_dh_cli/) example. + +The commands are defined by the `[project.scripts]` entry points in [`pyproject.toml`](pyproject.toml): + +```toml +[project.scripts] +my-dh-toolkit-query = "my_dh_toolkit.cli:app" +my-dh-toolkit-process = "my_dh_toolkit.processor:process" +``` ## Installation +From the repository root: + ```shell -pip install . +pip install ./my_dh_toolkit ``` Or in editable mode for development: ```shell -pip install -e . +pip install -e ./my_dh_toolkit ``` -## Usage as a Library - -> [!NOTE] -> All Deephaven functionality requires a running server. Start the server before importing Deephaven modules. - -Import and use the library functions in your Python code: +## Usage as command-line tools -```python -# Start the Deephaven server -from deephaven_server import Server -server = Server(port=10000, jvm_args=["-Xmx4g"]) -server.start() +Run the installed commands on the sample data. Each command starts its own Deephaven server, so no separate setup is needed: -# Now use the library functions -from my_dh_toolkit.queries import filter_by_threshold, add_computed_columns -from my_dh_toolkit import my_dh_query, batch_process -from deephaven import read_csv - -data = read_csv("../data/sample.csv") -filtered = filter_by_threshold(data, "Score", 75.0) - -# Or use the exported functions -result = my_dh_query("../data/sample.csv", verbose=True) +```shell +my-dh-toolkit-query data/sample.csv --verbose +my-dh-toolkit-process data/batch --output output --verbose ``` -## Usage as CLI Commands +`my-dh-toolkit-query` processes a single CSV file. `my-dh-toolkit-process` processes every CSV file in a directory and writes the results to the output directory. -Run the installed commands directly from a terminal. Both `my-dh-toolkit-query` and `my-dh-toolkit-process` start their own Deephaven server: +## Usage as a library -```shell -my-dh-toolkit-query ../data/sample.csv --verbose -my-dh-toolkit-process ../data/batch --output ./output --verbose -``` +> [!NOTE] +> All Deephaven functionality requires a running server in the same Python process. Start the server before importing `deephaven` modules. -The underlying functions are also importable, so you can call them directly within a Python session that already has a server running: +From the repository root, start Python and use the library: ```python -# Start the Deephaven server +# A Deephaven server must be running before deephaven modules are imported. from deephaven_server import Server -server = Server(port=10000, jvm_args=["-Xmx4g"]) -server.start() +Server(port=10000, jvm_args=["-Xmx4g"]).start() -# Call the underlying functions directly -from my_dh_toolkit import my_dh_query, batch_process -result = my_dh_query("../data/sample.csv", verbose=True) -batch_process("../data/batch/", "./output", verbose=True) +# Import and use the installed library. +from my_dh_toolkit.queries import filter_by_threshold, add_computed_columns +from deephaven import read_csv + +data = read_csv("data/sample.csv") +filtered = filter_by_threshold(data, "Score", 75.0) +enhanced = add_computed_columns(filtered) ``` -## Commands +## Command reference ### my-dh-toolkit-query -Process a single CSV file with Deephaven. +Process a single CSV file with Deephaven. The file must contain a `Score` column. **Arguments:** -- `input_file` - Path to the CSV file to process + +- `input_file` - Path to the CSV file to process. **Options:** -- `--verbose, -v` - Enable verbose output + +- `--verbose, -v` - Enable verbose output. ### my-dh-toolkit-process -Batch process multiple CSV files from a directory. +Batch process every CSV file in a directory. Each file must contain a `Score` column. **Arguments:** -- `directory` - Directory containing CSV files to process -**Options:** -- `--output, -o` - Output directory (default: ./output) -- `--verbose, -v` - Enable verbose output +- `directory` - Directory containing CSV files to process. -## Available Functions +**Options:** -### Query Functions (`my_dh_toolkit.queries`) +- `--output, -o` - Output directory (default: `./output`). +- `--verbose, -v` - Enable verbose output. -- `filter_by_threshold(table, column, threshold)` - Filter table rows where column value exceeds threshold -- `add_computed_columns(table)` - Add commonly used computed columns to a table -- `summarize_by_group(table, group_col, value_col)` - Create summary statistics grouped by a column +## Available functions -### Utility Functions (`my_dh_toolkit.utils`) +### Query functions (`my_dh_toolkit.queries`) -- `validate_columns(table, required_columns)` - Check if table has all required columns -- `get_table_info(table)` - Get basic information about a table +- `filter_by_threshold(table, column, threshold)` - Filter table rows where the column value exceeds the threshold. +- `add_computed_columns(table)` - Add commonly used computed columns to a table. +- `summarize_by_group(table, group_col, value_col)` - Create summary statistics grouped by a column. -### Exported Functions (`my_dh_toolkit`) +### Utility functions (`my_dh_toolkit.utils`) -- `my_dh_query(input_file, verbose)` - Read and process a CSV file -- `batch_process(directory, output_dir, verbose)` - Process multiple CSV files +- `validate_columns(table, required_columns)` - Check if a table has all required columns. +- `get_table_info(table)` - Get basic information about a table. ## Requirements - Python 3.9 or later +- Java 17 or later - Deephaven Server 0.35.0 or later - Click 8.0.0 or later diff --git a/my_dh_toolkit/src/my_dh_toolkit/__init__.py b/my_dh_toolkit/src/my_dh_toolkit/__init__.py index 31bf609..f0c1ff3 100644 --- a/my_dh_toolkit/src/my_dh_toolkit/__init__.py +++ b/my_dh_toolkit/src/my_dh_toolkit/__init__.py @@ -1,8 +1,9 @@ -"""My Deephaven package for data processing.""" +"""My Deephaven package for data processing. -__version__ = "0.1.0" - -from my_dh_toolkit.cli import my_dh_query -from my_dh_toolkit.processor import batch_process +This __init__ deliberately imports nothing that requires Deephaven: the CLI +entry points import this package before a Deephaven server is running, so the +package must be importable without one. The library API lives in the +`my_dh_toolkit.queries` and `my_dh_toolkit.utils` submodules. +""" -__all__ = ["my_dh_query", "batch_process"] +__version__ = "0.1.0" diff --git a/my_dh_toolkit/src/my_dh_toolkit/processor.py b/my_dh_toolkit/src/my_dh_toolkit/processor.py index 65da8b6..7560649 100644 --- a/my_dh_toolkit/src/my_dh_toolkit/processor.py +++ b/my_dh_toolkit/src/my_dh_toolkit/processor.py @@ -30,7 +30,10 @@ def batch_process(directory: str, output_dir: str, verbose: bool = False) -> Non if verbose: click.echo(f"Processing {csv_file.name}...") - table = read_csv(str(csv_file)) + try: + table = read_csv(str(csv_file)) + except Exception as e: + raise click.ClickException(f"Failed to read CSV file '{csv_file}': {e}") column_names = [col.name for col in table.columns] if "Score" not in column_names: diff --git a/setuptools-deployment.md b/setuptools-deployment.md index e5fea0c..63b769c 100644 --- a/setuptools-deployment.md +++ b/setuptools-deployment.md @@ -80,7 +80,7 @@ data = read_csv("data.csv") ## Packaging scenarios -Different projects have different needs. The example repository demonstrates three common scenarios: +Different projects have different needs. The example repository demonstrates three common scenarios. The Python usage snippets below assume a running Deephaven server, as shown in [Server initialization](#server-initialization). ### Library-only package @@ -169,6 +169,7 @@ my_dh_toolkit/ ```python # As a library from my_dh_toolkit.queries import filter_by_threshold +from deephaven import read_csv data = read_csv("data.csv") filtered = filter_by_threshold(data, "Score", 75.0) @@ -470,14 +471,15 @@ where = ["src"] Create `src/my_dh_toolkit/__init__.py`: ```python -"""My Deephaven package for data processing.""" - -__version__ = "0.1.0" +"""My Deephaven package for data processing. -from my_dh_toolkit.cli import my_dh_query -from my_dh_toolkit.processor import batch_process +This __init__ deliberately imports nothing that requires Deephaven: the CLI +entry points import this package before a Deephaven server is running, so the +package must be importable without one. The library API lives in the +`my_dh_toolkit.queries` and `my_dh_toolkit.utils` submodules. +""" -__all__ = ["my_dh_query", "batch_process"] +__version__ = "0.1.0" ``` Create `src/my_dh_toolkit/__main__.py`: @@ -528,7 +530,10 @@ def batch_process(directory: str, output_dir: str, verbose: bool = False) -> Non if verbose: click.echo(f"Processing {csv_file.name}...") - table = read_csv(str(csv_file)) + try: + table = read_csv(str(csv_file)) + except Exception as e: + raise click.ClickException(f"Failed to read CSV file '{csv_file}': {e}") column_names = [col.name for col in table.columns] if "Score" not in column_names: @@ -706,18 +711,7 @@ Entry-point commands like `my-dh-query` start their own Deephaven server, so the my-dh-query data.csv --verbose ``` -The underlying function is also importable, so you can call it directly within a Python session that already has a server running: - -```python -# Start the Deephaven server -from deephaven_server import Server -server = Server(port=10000, jvm_args=["-Xmx4g"]) -server.start() - -# Call the underlying function directly -from my_dh_cli.cli import my_dh_query -result = my_dh_query("data.csv", verbose=True) -``` +For programmatic use, install a library package (or the combined package) and import its functions as shown in [Use a library package](#use-a-library-package). ## Building and distributing From dfab129ceab5c6bfb38e9c00d92e658b2049852c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 11 Sep 2026 17:16:15 +0000 Subject: [PATCH 10/13] Clarify Example 3, surface sample data, document toml dependencies, sync guide Co-authored-by: chipkent <5250374+chipkent@users.noreply.github.com> --- README.md | 48 ++++++++++++++++++------------------ my_dh_cli/README.md | 2 +- my_dh_cli/pyproject.toml | 4 ++- my_dh_library/README.md | 2 +- my_dh_library/pyproject.toml | 1 + my_dh_toolkit/README.md | 4 +-- my_dh_toolkit/pyproject.toml | 2 ++ setuptools-deployment.md | 40 ++++++++++++++++++++---------- 8 files changed, 61 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index cdee4c8..03d9afa 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ This repository shows how to package Python code that uses [Deephaven Community | Example | Pattern | Installing it provides | |---|---|---| | [`my_dh_library/`](my_dh_library/) | Library only | Functions to import in Python code | -| [`my_dh_cli/`](my_dh_cli/) | Command-line tool only | A `my-dh-query` terminal command | -| [`my_dh_toolkit/`](my_dh_toolkit/) | Library and command-line tools combined | Importable functions plus `my-dh-toolkit-query` and `my-dh-toolkit-process` commands | +| [`my_dh_cli/`](my_dh_cli/) | Command line tool only | A `my-dh-query` terminal command | +| [`my_dh_toolkit/`](my_dh_toolkit/) | Library and command line tools combined | Importable functions plus `my-dh-toolkit-query` and `my-dh-toolkit-process` commands | `my_dh_toolkit` is the other two patterns merged into a single package: its library modules play the same role as `my_dh_library`, and its commands play the same role as `my_dh_cli`. @@ -14,7 +14,7 @@ All three examples follow the [Python Packaging User Guide](https://packaging.py ## Choose an example -- Start from **`my_dh_library`** to share reusable functions that other projects import. There is no command-line interface. +- Start from **`my_dh_library`** to share reusable functions that other projects import. There is no command line interface. - Start from **`my_dh_cli`** to ship a tool that users run from a terminal. No library code is exposed. - Start from **`my_dh_toolkit`** to provide both: importable functions for Python users and commands for terminal users. @@ -33,6 +33,13 @@ git clone https://github.com/deephaven-examples/deephaven-python-packaging.git cd deephaven-python-packaging ``` +## Sample data + +The examples read the CSV files in the `data/` directory: + +- `data/sample.csv` — a single 10-row file with `Name`, `Score`, `Value`, and `Category` columns. It is the input for the single-file examples: the library snippets, `my-dh-query`, and `my-dh-toolkit-query`. +- `data/batch/` — three smaller files (`file1.csv`, `file2.csv`, and `file3.csv`) with the same columns but different rows. It is the input for `my-dh-toolkit-process`, which processes every CSV file in a directory. + ## Example 1: `my_dh_library` — a library **The story:** package reusable Deephaven query functions so that other projects can `pip install` the package and import the functions. @@ -81,7 +88,7 @@ print(f"{filtered.size} of {data.size} rows have Score > 75") - [`queries.py`](my_dh_library/src/my_dh_library/queries.py) — plain functions that take and return Deephaven tables. - [`__init__.py`](my_dh_library/src/my_dh_library/__init__.py) — re-exports the public functions. -## Example 2: `my_dh_cli` — a command-line tool +## Example 2: `my_dh_cli` — a command line tool **The story:** package a Deephaven script as a terminal command. `pip install` creates a `my-dh-query` command that users run without writing any Python. @@ -120,27 +127,27 @@ The command starts its own Deephaven server, reads the CSV file, adds a computed - [`cli.py`](my_dh_cli/src/my_dh_cli/cli.py) — a [Click](https://click.palletsprojects.com/) command that starts the Deephaven server itself, so it works as a standalone tool. - [`__main__.py`](my_dh_cli/src/my_dh_cli/__main__.py) — allows `python -m my_dh_cli data/sample.csv` as an alternative during development. -## Example 3: `my_dh_toolkit` — a library and commands in one package +## Example 3: `my_dh_toolkit` — a library and command line tools in one package -**The story:** one package, two interfaces. Terminal users get commands; Python users import functions. The library modules (`queries.py`, `utils.py`) match `my_dh_library`, and the command modules (`cli.py`, `processor.py`) follow the same pattern as `my_dh_cli`. +**The story:** one package that provides both interfaces. Python users import its query functions, just as in `my_dh_library`; terminal users run its installed commands, just as in `my_dh_cli`. The library modules reuse the `my_dh_library` code, and the command modules reuse the `my_dh_cli` code plus a second command that shows one package installing multiple commands. ``` my_dh_toolkit/ ├── src/ │ └── my_dh_toolkit/ -│ ├── __init__.py # Kept minimal — see "What to study" below +│ ├── __init__.py # Intentionally contains no imports — see "What to study" │ ├── __main__.py -│ ├── cli.py # my-dh-toolkit-query command -│ ├── processor.py # my-dh-toolkit-process command -│ ├── queries.py # Library: query functions -│ └── utils.py # Library: table validation helpers -├── pyproject.toml # Declares two entry points +│ ├── cli.py # Implements my-dh-toolkit-query (same code as my_dh_cli) +│ ├── processor.py # Implements my-dh-toolkit-process +│ ├── queries.py # Library query functions (same code as my_dh_library) +│ └── utils.py # Library table helpers (same code as my_dh_library) +├── pyproject.toml # Declares both commands └── README.md ``` ### Try the commands -Install the package, then run the two commands: +Install the package, then run each command: ```shell pip install -e ./my_dh_toolkit @@ -148,11 +155,11 @@ my-dh-toolkit-query data/sample.csv --verbose my-dh-toolkit-process data/batch --output output --verbose ``` -`my-dh-toolkit-query` processes a single CSV file. `my-dh-toolkit-process` processes every CSV file in a directory and writes the results to the output directory. Both start their own Deephaven server. +`my-dh-toolkit-query` processes one CSV file. `my-dh-toolkit-process` processes every CSV file in a directory and writes one result file per input to the output directory. Like `my-dh-query` in the previous example, each command starts its own Deephaven server. ### Try the library -The same installed package is importable. As with any Deephaven library, start a server first: +The same installation also provides the library. In a Python session, start a Deephaven server, then import and use the query functions: ```python # A Deephaven server must be running before deephaven modules are imported. @@ -170,15 +177,8 @@ print(f"{filtered.size} of {data.size} rows have Score > 75") ### What to study -- [`pyproject.toml`](my_dh_toolkit/pyproject.toml) — `[project.scripts]` defines multiple commands for one package. -- [`__init__.py`](my_dh_toolkit/src/my_dh_toolkit/__init__.py) — deliberately imports nothing that requires Deephaven. The entry-point commands import the package before a server is running, so the library API stays in the `queries` and `utils` submodules, and the command modules defer their `deephaven` imports until after the server starts. This is the key structural difference from a library-only package. - -## Sample data - -The `data/` directory holds the inputs used by the examples above: - -- `data/sample.csv` — one 10-row file with `Name`, `Score`, `Value`, and `Category` columns. Input for the library snippets, `my-dh-query`, and `my-dh-toolkit-query`. -- `data/batch/file1.csv`, `file2.csv`, `file3.csv` — three separate files with the same columns but different rows. Input for `my-dh-toolkit-process`, which processes every CSV file in the directory. +- [`pyproject.toml`](my_dh_toolkit/pyproject.toml) — a single `[project.scripts]` section defines both commands. +- [`__init__.py`](my_dh_toolkit/src/my_dh_toolkit/__init__.py) — contains no imports, and that is deliberate. Importing any `deephaven` module fails unless a Deephaven server is already running in the process. When a command such as `my-dh-toolkit-query` starts, Python imports the `my_dh_toolkit` package before the command has started its server. If `__init__.py` imported the query functions, that import chain would reach `deephaven` and every command would fail at startup. Keeping `__init__.py` empty and importing the library from its submodules (`my_dh_toolkit.queries`, `my_dh_toolkit.utils`) avoids the problem. `my_dh_library` can safely re-export its functions from `__init__.py` because it has no commands: it is only ever imported after a server is running. ## Adapt an example for your own project diff --git a/my_dh_cli/README.md b/my_dh_cli/README.md index 80ae851..004a4d8 100644 --- a/my_dh_cli/README.md +++ b/my_dh_cli/README.md @@ -1,6 +1,6 @@ # My Deephaven CLI -An example of packaging a Deephaven script as a command-line tool. Installing this package creates one terminal command, `my-dh-query`. No library code is exposed — users of this package never write Python. +An example of packaging a Deephaven script as a command line tool. Installing this package creates one terminal command, `my-dh-query`. No library code is exposed — users of this package never write Python. The command is defined by the `[project.scripts]` entry point in [`pyproject.toml`](pyproject.toml): diff --git a/my_dh_cli/pyproject.toml b/my_dh_cli/pyproject.toml index 6e844c5..7890660 100644 --- a/my_dh_cli/pyproject.toml +++ b/my_dh_cli/pyproject.toml @@ -5,11 +5,13 @@ build-backend = "setuptools.build_meta" [project] name = "my_dh_cli" version = "0.1.0" -description = "Command-line tool for data processing" +description = "Command line tool for data processing" readme = "README.md" requires-python = ">=3.9" dependencies = [ + # deephaven-server also provides the deephaven module (through its deephaven-core dependency). "deephaven-server>=0.35.0", + # click implements the command line interface. "click>=8.0.0", ] diff --git a/my_dh_library/README.md b/my_dh_library/README.md index 86de540..ff17d0e 100644 --- a/my_dh_library/README.md +++ b/my_dh_library/README.md @@ -1,6 +1,6 @@ # My Deephaven Library -An example of packaging reusable Deephaven query functions as a library. Installing this package makes its functions importable from any Python code. There are no command-line tools — this package is only ever imported. +An example of packaging reusable Deephaven query functions as a library. Installing this package makes its functions importable from any Python code. There are no command line tools — this package is only ever imported. ## Installation diff --git a/my_dh_library/pyproject.toml b/my_dh_library/pyproject.toml index aee4a0b..0e1f738 100644 --- a/my_dh_library/pyproject.toml +++ b/my_dh_library/pyproject.toml @@ -9,6 +9,7 @@ description = "Reusable Deephaven query functions" readme = "README.md" requires-python = ">=3.9" dependencies = [ + # deephaven-server also provides the deephaven module (through its deephaven-core dependency). "deephaven-server>=0.35.0", ] diff --git a/my_dh_toolkit/README.md b/my_dh_toolkit/README.md index 0f785cf..3b14798 100644 --- a/my_dh_toolkit/README.md +++ b/my_dh_toolkit/README.md @@ -3,7 +3,7 @@ An example of one package with two interfaces: - **A library** — importable query functions, matching the [`my_dh_library`](../my_dh_library/) example. -- **Command-line tools** — two terminal commands, following the same pattern as the [`my_dh_cli`](../my_dh_cli/) example. +- **Command line tools** — two terminal commands, following the same pattern as the [`my_dh_cli`](../my_dh_cli/) example. The commands are defined by the `[project.scripts]` entry points in [`pyproject.toml`](pyproject.toml): @@ -27,7 +27,7 @@ Or in editable mode for development: pip install -e ./my_dh_toolkit ``` -## Usage as command-line tools +## Usage as command line tools Run the installed commands on the sample data. Each command starts its own Deephaven server, so no separate setup is needed: diff --git a/my_dh_toolkit/pyproject.toml b/my_dh_toolkit/pyproject.toml index 1051e34..ebdd348 100644 --- a/my_dh_toolkit/pyproject.toml +++ b/my_dh_toolkit/pyproject.toml @@ -9,7 +9,9 @@ description = "Deephaven library and CLI tools" readme = "README.md" requires-python = ">=3.9" dependencies = [ + # deephaven-server also provides the deephaven module (through its deephaven-core dependency). "deephaven-server>=0.35.0", + # click implements the command line interfaces. "click>=8.0.0", ] diff --git a/setuptools-deployment.md b/setuptools-deployment.md index 63b769c..1117298 100644 --- a/setuptools-deployment.md +++ b/setuptools-deployment.md @@ -3,12 +3,12 @@ title: Packaging custom code and dependencies sidebar_label: Python packaging --- -[Python packaging](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/) enables you to create distributable packages containing custom code, command-line tools, and managed dependencies. Deephaven's pip-installable packages integrate seamlessly with modern Python packaging tools, allowing you to build reusable libraries and executable scripts that leverage Deephaven's query engine. This guide walks through the concepts and patterns for packaging Deephaven-based Python projects. +[Python packaging](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/) enables you to create distributable packages containing custom code, command line tools, and managed dependencies. Deephaven's pip-installable packages integrate seamlessly with modern Python packaging tools, allowing you to build reusable libraries and executable scripts that leverage Deephaven's query engine. This guide walks through the concepts and patterns for packaging Deephaven-based Python projects. Python packaging with [`pyproject.toml`](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/) provides: - **Reusable libraries** - Package query functions and utilities for import by other projects. -- **Command-line tools** - Build executable scripts with entry point definitions. +- **Command line tools** - Build executable scripts with entry point definitions. - **Dependency management** - Automatically install Deephaven and required packages. - **Distribution** - Share code as wheel archives via PyPI or direct distribution. - **Version control** - Specify compatible dependency versions for reproducible installations. @@ -27,12 +27,12 @@ cd deephaven-python-packaging The repository contains three example packages: - `my_dh_library/` - Library-only package with reusable query functions. -- `my_dh_cli/` - CLI-only package with command-line tools. +- `my_dh_cli/` - CLI-only package with command line tools. - `my_dh_toolkit/` - Combined package with both library and CLI functionality. ## Package structure -Modern Python packages use the **src-layout**, which is the recommended structure by the [Python Packaging Authority](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/). This layout keeps source code separate from tests and configuration files: +Modern Python packages use the **src-layout**, the structure [recommended by the Python Packaging Authority](https://packaging.python.org/en/latest/discussions/src-layout-vs-flat-layout/). This layout keeps source code separate from tests and configuration files: ``` my_dh_project/ @@ -76,7 +76,7 @@ data = read_csv("data.csv") - Each Python process has its own JVM. - Starting a server in one terminal doesn't help another terminal. - Entry-point CLI commands should start their own server internally (see [Use CLI functions](#use-cli-functions)) so they work standalone; only functions imported directly need an already-running session. -- The server uses approximately 4GB of memory by default (configurable via `jvm_args`). +- The examples size the JVM to 4 GB with `jvm_args=["-Xmx4g"]`; adjust this value to fit the workload. ## Packaging scenarios @@ -112,12 +112,12 @@ filtered = filter_by_threshold(data, "Score", 75.0) **Use when:** - Creating reusable utilities for other projects. -- You don't need a command-line interface. +- You don't need a command line interface. - Code will be imported, not executed directly. ### CLI-only package -Package executable command-line tools without exposing library code. +Package executable command line tools without exposing library code. **Structure:** @@ -140,13 +140,13 @@ my-dh-query data.csv --verbose **Use when:** -- Building command-line tools for data processing +- Building command line tools for data processing - You need clean function interfaces - You don't need to expose library code to other projects ### Combined package -Package both reusable library code and command-line tools. +Package both reusable library code and command line tools. **Structure:** @@ -217,6 +217,7 @@ description = "Reusable Deephaven query functions" readme = "README.md" requires-python = ">=3.9" dependencies = [ + # deephaven-server also provides the deephaven module (through its deephaven-core dependency). "deephaven-server>=0.35.0", ] @@ -241,6 +242,8 @@ Create `src/my_dh_library/utils.py`: ```python """Utility functions for working with Deephaven tables.""" +from __future__ import annotations + from deephaven.table import Table @@ -341,11 +344,13 @@ build-backend = "setuptools.build_meta" [project] name = "my_dh_cli" version = "0.1.0" -description = "Command-line tool for data processing" +description = "Command line tool for data processing" readme = "README.md" requires-python = ">=3.9" dependencies = [ + # deephaven-server also provides the deephaven module (through its deephaven-core dependency). "deephaven-server>=0.35.0", + # click implements the command line interface. "click>=8.0.0", ] @@ -456,7 +461,9 @@ description = "Deephaven library and CLI tools" readme = "README.md" requires-python = ">=3.9" dependencies = [ + # deephaven-server also provides the deephaven module (through its deephaven-core dependency). "deephaven-server>=0.35.0", + # click implements the command line interfaces. "click>=8.0.0", ] @@ -482,6 +489,8 @@ package must be importable without one. The library API lives in the __version__ = "0.1.0" ``` +The empty `__init__.py` is the key structural difference from a library-only package: if it imported the query functions, the import chain would reach `deephaven` and both commands would fail before they could start their server. + Create `src/my_dh_toolkit/__main__.py`: ```python @@ -596,6 +605,7 @@ description = "Reusable Deephaven query functions" readme = "README.md" requires-python = ">=3.9" dependencies = [ + # deephaven-server also provides the deephaven module (through its deephaven-core dependency). "deephaven-server>=0.35.0", ] @@ -618,7 +628,7 @@ For CLI packages, add a `[project.scripts]` section: my-dh-query = "my_dh_cli.cli:app" ``` -This creates a command-line entry point that calls the `app` function from `my_dh_cli.cli`. +This creates a command line entry point that calls the `app` function from `my_dh_cli.cli`. ## Managing dependencies @@ -633,6 +643,8 @@ dependencies = [ ] ``` +Declaring `deephaven-server` is sufficient for Deephaven: it depends on a matching version of `deephaven-core`, which provides the `deephaven` module that packages import. + ### Version constraints Use version specifiers to control which versions are acceptable: @@ -737,6 +749,7 @@ This creates a `.whl` file in `dist/` that can be: - Keep package names lowercase with underscores - Match the package directory name to the import name - Include `__init__.py` in all package directories +- In packages that define entry-point commands, keep `__init__.py` free of imports that require a running Deephaven server ### Dependencies @@ -767,15 +780,16 @@ Each example includes: - Complete source code - Configured `pyproject.toml` -- Sample data files - Comprehensive README - Usage examples +The repository also provides shared sample data in its `data/` directory for trying the examples. + ## Related documentation - [Install and use Python packages](https://deephaven.io/core/docs/how-to-guides/install-and-use-python-packages/) - [Use the Deephaven Python package](https://deephaven.io/core/docs/how-to-guides/deephaven-python-package/) - [Python Packaging User Guide](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/) -- [Creating command-line tools](https://packaging.python.org/en/latest/guides/creating-command-line-tools/) +- [Creating command line tools](https://packaging.python.org/en/latest/guides/creating-command line-tools/) - [Setuptools documentation](https://setuptools.pypa.io/) - [Click documentation](https://click.palletsprojects.com/) From c8d8dff4cac33b29bf7b932acb96095e9dcb8e0a Mon Sep 17 00:00:00 2001 From: Chip Kent <5250374+chipkent@users.noreply.github.com> Date: Fri, 11 Sep 2026 11:37:06 -0600 Subject: [PATCH 11/13] Fix link for creating command-line tools Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- setuptools-deployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setuptools-deployment.md b/setuptools-deployment.md index 1117298..e66dd3f 100644 --- a/setuptools-deployment.md +++ b/setuptools-deployment.md @@ -790,6 +790,6 @@ The repository also provides shared sample data in its `data/` directory for try - [Install and use Python packages](https://deephaven.io/core/docs/how-to-guides/install-and-use-python-packages/) - [Use the Deephaven Python package](https://deephaven.io/core/docs/how-to-guides/deephaven-python-package/) - [Python Packaging User Guide](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/) -- [Creating command line tools](https://packaging.python.org/en/latest/guides/creating-command line-tools/) +- [Creating and packaging command-line tools](https://packaging.python.org/en/latest/guides/creating-command-line-tools/) - [Setuptools documentation](https://setuptools.pypa.io/) - [Click documentation](https://click.palletsprojects.com/) From d6fb7910d285a9acd264086013ea255602db4df8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 11 Sep 2026 17:39:56 +0000 Subject: [PATCH 12/13] Reject identical batch input and output directories Co-authored-by: chipkent <5250374+chipkent@users.noreply.github.com> --- my_dh_toolkit/src/my_dh_toolkit/processor.py | 9 +++++++-- setuptools-deployment.md | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/my_dh_toolkit/src/my_dh_toolkit/processor.py b/my_dh_toolkit/src/my_dh_toolkit/processor.py index 7560649..459b053 100644 --- a/my_dh_toolkit/src/my_dh_toolkit/processor.py +++ b/my_dh_toolkit/src/my_dh_toolkit/processor.py @@ -4,8 +4,6 @@ def batch_process(directory: str, output_dir: str, verbose: bool = False) -> None: """Process multiple CSV files from a directory.""" - from deephaven import read_csv, write_csv - input_path = Path(directory) output_path = Path(output_dir) @@ -21,6 +19,13 @@ def batch_process(directory: str, output_dir: str, verbose: bool = False) -> Non except OSError as e: raise click.ClickException(f"Failed to create output directory '{output_path}': {e}") + if input_path.resolve() == output_path.resolve(): + raise click.ClickException( + f"Input and output directories must be different: '{input_path}'" + ) + + from deephaven import read_csv, write_csv + csv_files = list(input_path.glob("*.csv")) if verbose: diff --git a/setuptools-deployment.md b/setuptools-deployment.md index e66dd3f..898f948 100644 --- a/setuptools-deployment.md +++ b/setuptools-deployment.md @@ -513,8 +513,6 @@ from pathlib import Path def batch_process(directory: str, output_dir: str, verbose: bool = False) -> None: """Process multiple CSV files from a directory.""" - from deephaven import read_csv, write_csv - input_path = Path(directory) output_path = Path(output_dir) @@ -530,6 +528,13 @@ def batch_process(directory: str, output_dir: str, verbose: bool = False) -> Non except OSError as e: raise click.ClickException(f"Failed to create output directory '{output_path}': {e}") + if input_path.resolve() == output_path.resolve(): + raise click.ClickException( + f"Input and output directories must be different: '{input_path}'" + ) + + from deephaven import read_csv, write_csv + csv_files = list(input_path.glob("*.csv")) if verbose: From 7173ab4fb2f43b81c11cbe9257c431014270563b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 11 Sep 2026 17:43:24 +0000 Subject: [PATCH 13/13] Clarify packaging guide links and examples Co-authored-by: chipkent <5250374+chipkent@users.noreply.github.com> --- setuptools-deployment.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/setuptools-deployment.md b/setuptools-deployment.md index 898f948..a41da7d 100644 --- a/setuptools-deployment.md +++ b/setuptools-deployment.md @@ -32,7 +32,7 @@ The repository contains three example packages: ## Package structure -Modern Python packages use the **src-layout**, the structure [recommended by the Python Packaging Authority](https://packaging.python.org/en/latest/discussions/src-layout-vs-flat-layout/). This layout keeps source code separate from tests and configuration files: +The example packages in this guide use the **src-layout** described in the Python Packaging Authority's [src layout vs flat layout discussion](https://packaging.python.org/en/latest/discussions/src-layout-vs-flat-layout/). This layout keeps source code separate from tests and configuration files: ``` my_dh_project/ @@ -178,7 +178,7 @@ filtered = filter_by_threshold(data, "Score", 75.0) ```bash # As CLI commands my-dh-toolkit-query data.csv --verbose -my-dh-toolkit-process data/ --output results/ --verbose +my-dh-toolkit-process data/batch/ --output results/ --verbose ``` **Use when:** @@ -750,7 +750,7 @@ This creates a `.whl` file in `dist/` that can be: ### Package structure -- Use the src-layout for all packages +- Prefer the src-layout for packages like these examples - Keep package names lowercase with underscores - Match the package directory name to the import name - Include `__init__.py` in all package directories @@ -794,7 +794,8 @@ The repository also provides shared sample data in its `data/` directory for try - [Install and use Python packages](https://deephaven.io/core/docs/how-to-guides/install-and-use-python-packages/) - [Use the Deephaven Python package](https://deephaven.io/core/docs/how-to-guides/deephaven-python-package/) -- [Python Packaging User Guide](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/) +- [Writing your `pyproject.toml`](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/) +- [src layout vs flat layout](https://packaging.python.org/en/latest/discussions/src-layout-vs-flat-layout/) - [Creating and packaging command-line tools](https://packaging.python.org/en/latest/guides/creating-command-line-tools/) - [Setuptools documentation](https://setuptools.pypa.io/) - [Click documentation](https://click.palletsprojects.com/)