# PySpark dependencies (required)
py4j

# PySpark dependencies (optional)
numpy
pyarrow<13.0.0
pandas
scipy
plotly
mlflow>=2.3.1
scikit-learn
matplotlib
memory-profiler==0.60.0

# PySpark test dependencies
unittest-xml-reporting
openpyxl

# PySpark test dependencies (optional)
coverage

# Linter
mypy==0.982
pytest-mypy-plugins==1.9.3
flake8==3.9.0
# See SPARK-38680.
pandas-stubs<1.2.0.54

# Documentation (SQL)
mkdocs

# Documentation (Python)
pydata_sphinx_theme
ipython
nbsphinx
numpydoc
jinja2<3.0.0
sphinx<3.1.0
sphinx-plotly-directive
sphinx-copybutton<0.5.3
docutils<0.18.0
# See SPARK-38279.
markupsafe==2.0.1

# Development scripts
jira
PyGithub

# pandas API on Spark Code formatter.
black==22.6.0
py

# Spark Connect (required)
grpcio>=1.48,<1.57
grpcio-status>=1.48,<1.57
protobuf==3.20.3
googleapis-common-protos==1.56.4

# Spark Connect python proto generation plugin (optional)
mypy-protobuf==3.3.0
googleapis-common-protos-stubs==2.2.0
grpc-stubs==1.24.11

# TorchDistributor dependencies
torch
torchvision
torcheval

# DeepspeedTorchDistributor dependencies
deepspeed; sys_platform != 'darwin'

