wip: Make weaviatest a pip package

Enable the option to install weaviatest as a pip package
in order to facilitate the consumption from other repositories
and maximize the resuability

Signed-off-by: Rodrigo Lopez <[email protected]>

Dockerfile

@@ -1,23 +1,8 @@
-# Step 1: Use an official Python runtime as a parent image
-FROM python:3.10-slim
+FROM python:3.12-slim
 
-# Step 2: Set the working directory in the container
 WORKDIR /app
 
-# Step 3: Copy the requirements file to the container
-COPY requirements.txt .
-
-# Step 4: Install the required Python packages
-RUN pip install --no-cache-dir -r requirements.txt
-
-# Step 5: Copy the rest of the application code to the container
 COPY . .
+RUN pip install -e .
 
-# Step 6: Set the default entry point for the container
-# Use the `cli.py` script as the entry point for the container
 ENTRYPOINT ["python", "weaviatest.py"]
-
-# Optionally, provide a default command to the entrypoint
-# CMD ["--help"]
-
-

Makefile

@@ -0,0 +1,33 @@
+.PHONY: default env_install test build_container clean
+
+ENV := .venv
+SYSTEM_PYTHON := python3
+PYTHON_VERSION := 3.12.5
+ENV_PYTHON := ${ENV}/bin/python3
+ENV_SITE_PACKAGES := ${ENV}/lib/python${PYTHON_VERSION}/site-packages
+PROJECT_PACKAGE_NAME := eco_pytests
+ENV_PROJECT_PACKAGE_PATH := ${ENV_SITE_PACKAGES}/${PROJECT_PACKAGE_NAME}-%
+CONTAINER_NAME := weaviatest
+VERSION ?= latest #$(shell git rev-parse --short HEAD)
+PUSH_TAG ?= latest
+
+default: test
+
+${ENV_PYTHON}:
+	@echo "Creating Python ${PYTHON_VERSION} environment..." >&2
+	@${SYSTEM_PYTHON} -m venv ${ENV}
+	@${ENV_PYTHON} -m pip install -U pip setuptools
+
+${ENV_PROJECT_PACKAGE_PATH}: ${ENV_PYTHON}
+	@echo "Installing Python project in ${ENV_SITE_PACKAGES}"
+	@${ENV_PYTHON} -m pip install .
+
+env_install: ${ENV_PROJECT_PACKAGE_PATH}
+
+test: ${ENV_PROJECT_PACKAGE_PATH}
+
+build_container:
+	docker build -t ${CONTAINER_NAME}:${VERSION} .
+
+clean:
+	@rm -rf ${ENV}

README.md

@@ -35,26 +35,52 @@ The Weaviatest CLI provides the following actions:
 
 You can run the Weaviatest CLI directly from the command line.
 
-## Requirements
+## Install
 
-The Weaviatest CLI requires the following dependencies, which are listed in the `requirements.txt` file. If the required client changes are not yet published, you can use a branch from the `weaviate-python-client` repository. To do so, add the following requirement to the `requirements.txt` file:
+Weaviatest can be installed as a pip package to do so, you can run the command:
+``` bash
+pip install -U pip setuptools
+```
+
+As an alternative, the pip package can be install with the flag -e to make the installation package editable
+``` bash
+pip install -e -U pip setuptools
+```
 
+Finally, there is a Makefile to facilitate the creation of a python environment and install the weaviatest package:
+``` bash
+make env_install
+source .venv/bin/activate
 ```
+
+### Requirements
+
+The Weaviatest CLI requires the following dependencies, which are listed in the `setup.cfg` file. If the required client changes are not yet published, you can use a branch from the `weaviate-python-client` repository. To do so, add the following requirement to the `setup.cfg` file:
+
+```bash
 weaviate-client @ git+https://github.com/weaviate/weaviate-python-client.git@<branch>
 ```
 
 For example:
 
-```
+```bash
 weaviate-client @ git+https://github.com/weaviate/weaviate-python-client.git@dev/1.26
 ```
 
 Then, run the following command to install the dependencies:
 
+```bash
+pip install -U pip setuptools
 ```
-pip install -r requirements.txt
+
+#### Dev requirements
+
+For development, there are extra requirements to be installed. They can be install once the virtual environment has been activated, and the run:
+``` bash
+pip install -r dev-requirements.txt
 ```
 
+
 ### Linting
 
 You can configure the [pre-commit](https://pre-commit.com/) to automatically run all linters locally on each commit. Install `pre-commit` on your system, and then enable it with `pre-commit install`.

dev-requirements.txt

@@ -0,0 +1,3 @@
+pre-commit
+flake8==7.1.1
+black==24.4.2

pyproject.toml

@@ -0,0 +1,7 @@
+[build-system]
+requires = [
+    "setuptools >= 65",
+    "setuptools_scm[toml] >6.2",
+    "wheel >= 0.38.1",
+]
+build-backend = 'setuptools.build_meta'

setup.cfg

@@ -0,0 +1,55 @@
+[metadata]
+name = weaviatests
+provides = weaviatests
+description = Weaviate library for testing
+url = https://github.com/weaviate/weaviatest
+download_url = https://github.com/weaviate/weaviatest
+author = weaviate
+author_email =
+keywords = python3, sql
+license = GPL-3
+platforms = Linux
+version = 0.0.1
+project_urls =
+    Docs = https://github.com/weaviate/weaviatest/blob/main/README.md
+    Bugs = https://github.com/weaviate/weaviatest/issues
+
+license_files =
+    LICENSE
+long_description = file: README.md
+long_description_content_type = text/markdown
+classifiers =
+    Development Status :: 3 - Alpha
+    Environment :: Console
+    Intended Audience :: Developers
+    Intended Audience :: Other Audience
+    Natural Language :: English
+    License :: OSI Approved :: GNU General Public License (GPL)
+    License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)
+    Operating System :: POSIX :: Linux
+    Programming Language :: Python
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3 :: Only
+    Programming Language :: Python :: 3.12
+    Programming Language :: Python :: Implementation :: CPython
+    Topic :: Software Development
+
+[options]
+include_package_data = True
+python_requires =
+    >=3.12
+install_requires =
+    weaviate-client
+    numpy
+    semver
+    click
+setup_requires =
+    pip
+    setuptools
+zip_safe = False
+packages =
+    weaviatest
+
+[options.package_data]
+# If any package or subpackage contains *.txt, *.rst or *.md files, include them:
+*: ["*.txt", "*.rst", "*.md", LICENSE],

setup.py

@@ -0,0 +1,3 @@
+from setuptools import setup
+
+setup()

weaviatest.py

@@ -3,21 +3,21 @@
 import click
 import sys
 
-from lib.create_collection import create_collection
-from lib.create_tenants import create_tenants
-from lib.create_backup import create_backup
-from lib.delete_collection import delete_collection
-from lib.delete_data import delete_data
-from lib.delete_tenants import delete_tenants
-from lib.create_data import ingest_data
-from lib.update_collection import update_collection
-from lib.update_data import update_data
-from lib.update_tenants import update_tenants
-from lib.query_data import query_data
-from lib.restore_backup import restore_backup
-from lib.get_collection import get_collection
-from lib.get_tenants import get_tenants
-from lib.common import connect_to_weaviate
+from weaviatest.create_collection import create_collection
+from weaviatest.create_tenants import create_tenants
+from weaviatest.create_backup import create_backup
+from weaviatest.delete_collection import delete_collection
+from weaviatest.delete_data import delete_data
+from weaviatest.delete_tenants import delete_tenants
+from weaviatest.create_data import ingest_data
+from weaviatest.update_collection import update_collection
+from weaviatest.update_data import update_data
+from weaviatest.update_tenants import update_tenants
+from weaviatest.query_data import query_data
+from weaviatest.restore_backup import restore_backup
+from weaviatest.get_collection import get_collection
+from weaviatest.get_tenants import get_tenants
+from weaviatest.common import connect_to_weaviate
 
 
 # General CLI group for Weaviate operations

weaviatest/__init__.py

@@ -0,0 +1 @@
+# __init__.py

lib/create_data.py → weaviatest/create_data.py

@@ -1,4 +1,4 @@
-import lib.common as common
+import weaviatest.common as common
 import json
 import numpy as np
 import random

lib/delete_tenants.py → weaviatest/delete_tenants.py

@@ -1,4 +1,4 @@
-import lib.common as common
+import weaviatest.common as common
 import semver
 from weaviate.collections.classes.tenants import Tenant
 

lib/get_tenants.py → weaviatest/get_tenants.py

@@ -31,3 +31,4 @@ def get_tenants(client, collection, verbose):
         print(
             f"{len(tenants):<20}{len(inactive_tenants):<20}{len(active_tenants):<20}{len(offoaded_tenants):<20}"
         )
+    return tenants

lib/query_data.py → weaviatest/query_data.py

@@ -1,4 +1,4 @@
-import lib.common as common
+import weaviatest.common as common
 import weaviate.classes.config as wvc
 from weaviate.classes.query import MetadataQuery
 from weaviate.collections.classes.tenants import TenantActivityStatus

lib/update_data.py → weaviatest/update_data.py

@@ -1,4 +1,4 @@
-import lib.common as common
+import weaviatest.common as common
 import numpy as np
 import random
 import weaviate.classes.config as wvc