Metadata-Version: 2.4
Name: insyt-secure
Version: 0.3.1
Summary: A proprietary service for secure database and API interactions
Author-email: "Insyt.co" <support@insyt.co>
License: Proprietary
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3 :: Only
Classifier: Programming Language :: Python :: 3.9
Classifier: Programming Language :: Python :: 3.10
Classifier: License :: Other/Proprietary License
Classifier: Operating System :: OS Independent
Requires-Python: >=3.9
Description-Content-Type: text/markdown
Requires-Dist: paho-mqtt==1.6.1
Requires-Dist: requests==2.31.0
Requires-Dist: python-json-logger==2.0.7
Requires-Dist: httpx==0.25.1
Requires-Dist: tenacity==8.2.3
Requires-Dist: backoff==2.2.1
Requires-Dist: python-dateutil==2.8.2
Requires-Dist: pytz==2023.3
Requires-Dist: protobuf<6.0.0dev,>=3.19.5
Requires-Dist: numpy==2.2.3
Requires-Dist: pandas==2.2.3
Requires-Dist: scipy==1.15.2
Requires-Dist: scikit-learn==1.6.1
Requires-Dist: rapidfuzz==3.5.2
Requires-Dist: statsmodels==0.14.4
Requires-Dist: sympy==1.12
Requires-Dist: polars==0.19.12
Requires-Dist: scikit-survival==0.24.0
Requires-Dist: featuretools==1.27.0
Requires-Dist: numexpr>=2.8.7
Requires-Dist: bottleneck>=1.3.7
Requires-Dist: pyarrow>=14.0.1
Requires-Dist: orjson>=3.9.10
Requires-Dist: thriftpy2>=0.4.16
Requires-Dist: grpcio-tools>=1.59.0
Provides-Extra: rdbms
Requires-Dist: asyncpg>=0.29.0; extra == "rdbms"
Requires-Dist: pg8000==1.29.4; extra == "rdbms"
Requires-Dist: aiomysql>=0.2.0; extra == "rdbms"
Requires-Dist: pymysql==1.1.0; extra == "rdbms"
Requires-Dist: mysql-connector-python>=8.0.31; extra == "rdbms"
Requires-Dist: sqlalchemy==2.0.23; extra == "rdbms"
Requires-Dist: alembic==1.12.1; extra == "rdbms"
Requires-Dist: oracledb>=1.4.1; extra == "rdbms"
Requires-Dist: pymssql>=2.2.7; extra == "rdbms"
Requires-Dist: pyodbc>=4.0.39; extra == "rdbms"
Requires-Dist: clickhouse-driver>=0.2.6; extra == "rdbms"
Requires-Dist: snowflake-connector-python>=3.5.0; extra == "rdbms"
Requires-Dist: duckdb>=0.9.1; extra == "rdbms"
Provides-Extra: postgres
Requires-Dist: asyncpg>=0.29.0; extra == "postgres"
Requires-Dist: pg8000==1.29.4; extra == "postgres"
Provides-Extra: mysql
Requires-Dist: aiomysql>=0.2.0; extra == "mysql"
Requires-Dist: pymysql==1.1.0; extra == "mysql"
Requires-Dist: mysql-connector-python>=8.0.31; extra == "mysql"
Provides-Extra: mssql
Requires-Dist: pymssql>=2.2.7; extra == "mssql"
Provides-Extra: oracle
Requires-Dist: oracledb>=1.4.1; extra == "oracle"
Provides-Extra: clickhouse
Requires-Dist: clickhouse-driver>=0.2.6; extra == "clickhouse"
Provides-Extra: snowflake
Requires-Dist: snowflake-connector-python>=3.5.0; extra == "snowflake"
Provides-Extra: duckdb
Requires-Dist: duckdb>=0.9.1; extra == "duckdb"
Provides-Extra: nosql
Requires-Dist: pymongo==4.6.0; extra == "nosql"
Requires-Dist: motor>=3.3.2; extra == "nosql"
Requires-Dist: redis==5.0.1; extra == "nosql"
Requires-Dist: cassandra-driver==3.28.0; extra == "nosql"
Requires-Dist: scylla-driver==3.26.3; extra == "nosql"
Requires-Dist: neo4j==5.14.1; extra == "nosql"
Requires-Dist: elasticsearch==8.11.0; extra == "nosql"
Requires-Dist: couchdb==1.2.0; extra == "nosql"
Requires-Dist: happybase==1.2.0; extra == "nosql"
Requires-Dist: pysolr==3.9.0; extra == "nosql"
Provides-Extra: mongodb
Requires-Dist: pymongo==4.6.0; extra == "mongodb"
Requires-Dist: motor>=3.3.2; extra == "mongodb"
Provides-Extra: redis
Requires-Dist: redis==5.0.1; extra == "redis"
Provides-Extra: cassandra
Requires-Dist: cassandra-driver==3.28.0; extra == "cassandra"
Provides-Extra: neo4j
Requires-Dist: neo4j==5.14.1; extra == "neo4j"
Provides-Extra: elasticsearch
Requires-Dist: elasticsearch==8.11.0; extra == "elasticsearch"
Provides-Extra: couchdb
Requires-Dist: couchdb==1.2.0; extra == "couchdb"
Provides-Extra: happybase
Requires-Dist: happybase==1.2.0; extra == "happybase"
Provides-Extra: pysolr
Requires-Dist: pysolr==3.9.0; extra == "pysolr"
Provides-Extra: vector
Requires-Dist: pinecone-client==2.2.4; extra == "vector"
Requires-Dist: qdrant-client==1.13.3; extra == "vector"
Requires-Dist: milvus==2.2.16; extra == "vector"
Requires-Dist: weaviate-client==3.24.1; extra == "vector"
Requires-Dist: pyvespa==0.36.0; extra == "vector"
Requires-Dist: chromadb>=0.4.17; extra == "vector"
Requires-Dist: faiss-cpu>=1.7.4; extra == "vector"
Provides-Extra: pinecone
Requires-Dist: pinecone-client==2.2.4; extra == "pinecone"
Provides-Extra: qdrant
Requires-Dist: qdrant-client==1.13.3; extra == "qdrant"
Provides-Extra: milvus
Requires-Dist: milvus==2.2.16; extra == "milvus"
Provides-Extra: weaviate
Requires-Dist: weaviate-client==3.24.1; extra == "weaviate"
Provides-Extra: pyvespa
Requires-Dist: pyvespa==0.36.0; extra == "pyvespa"
Provides-Extra: chroma
Requires-Dist: chromadb>=0.4.17; extra == "chroma"
Provides-Extra: faiss
Requires-Dist: faiss-cpu>=1.7.4; extra == "faiss"
Provides-Extra: cloud
Requires-Dist: boto3==1.33.6; extra == "cloud"
Requires-Dist: google-cloud-storage==2.13.0; extra == "cloud"
Requires-Dist: google-cloud-bigquery==3.11.4; extra == "cloud"
Requires-Dist: google-cloud-firestore==2.13.1; extra == "cloud"
Requires-Dist: azure-cosmos==4.5.1; extra == "cloud"
Requires-Dist: azure-storage-blob==12.19.0; extra == "cloud"
Requires-Dist: azure-data-tables==12.4.4; extra == "cloud"
Provides-Extra: aws
Requires-Dist: boto3==1.33.6; extra == "aws"
Requires-Dist: google-cloud-storage==2.13.0; extra == "aws"
Requires-Dist: google-cloud-bigquery==3.11.4; extra == "aws"
Requires-Dist: google-cloud-firestore==2.13.1; extra == "aws"
Provides-Extra: azure
Requires-Dist: azure-cosmos==4.5.1; extra == "azure"
Requires-Dist: azure-storage-blob==12.19.0; extra == "azure"
Requires-Dist: azure-data-tables==12.4.4; extra == "azure"
Provides-Extra: messaging
Requires-Dist: confluent-kafka>=2.3.0; extra == "messaging"
Requires-Dist: pulsar-client>=3.6.1; extra == "messaging"
Requires-Dist: pika==1.3.1; extra == "messaging"
Requires-Dist: nats-py==2.6.0; extra == "messaging"
Requires-Dist: stomp.py==8.1.0; extra == "messaging"
Requires-Dist: google-cloud-pubsub==2.18.4; extra == "messaging"
Provides-Extra: kafka
Requires-Dist: confluent-kafka>=2.3.0; extra == "kafka"
Provides-Extra: pulsar
Requires-Dist: pulsar-client>=3.6.1; extra == "pulsar"
Provides-Extra: rabbitmq
Requires-Dist: pika==1.3.1; extra == "rabbitmq"
Provides-Extra: all
Requires-Dist: insyt-secure[cloud,messaging,nosql,rdbms,vector]; extra == "all"
Provides-Extra: all-performance
Requires-Dist: insyt-secure[all,performance]; extra == "all-performance"

# Insyt Secure

A proprietary service for securely interacting with databases, APIs, and other services within a secure environment. Visit https://insyt.co for more information.

## Quick Start

### Installation

```bash
# Basic installation (core functionality only)
pip install insyt-secure
```

#### Advanced Installation Options

Insyt Secure provides flexible installation options to include only the dependencies you need:

```bash
# Install with PostgreSQL support
pip install "insyt-secure[postgres]"

# Install with MongoDB support
pip install "insyt-secure[mongodb]"

# Install with multiple database support
pip install "insyt-secure[postgres,mongodb,redis]"

# Install with vector database support
pip install "insyt-secure[pinecone]" # or any other vector DB

# Install with cloud provider support
pip install "insyt-secure[aws]" # or azure

# Install with messaging system support
pip install "insyt-secure[kafka]" # or rabbitmq, pulsar

# Complete installation with all dependencies
pip install "insyt-secure[all]"
```

Available extension categories:
- SQL databases: `postgres`, `mysql`, `mssql`, `oracle`, `clickhouse`, `snowflake`, `duckdb`
- NoSQL databases: `mongodb`, `redis`, `cassandra`, `neo4j`, `elasticsearch`, `couchdb`
- Vector databases: `pinecone`, `qdrant`, `milvus`, `weaviate`, `chroma`, `faiss`
- Cloud services: `aws`, `azure`
- Messaging systems: `kafka`, `pulsar`, `rabbitmq`

Broader categories are also available: `rdbms`, `nosql`, `vector`, `cloud`, `messaging`

### Basic Usage

```bash
# Run with a single project
insyt-secure --projects "your-project-id-123:your-api-key-xyz"
```

### Multi-Project Support

Insyt Secure supports managing one or more projects simultaneously:

```bash
# Run with multiple projects
insyt-secure --projects "project-id-1:api-key-1,project-id-2:api-key-2"
```

Each project connection is managed independently, allowing for simultaneous interactions with multiple Insyt projects, each with its own topic subscriptions and credential management.

### Getting Help

```bash
# View all available options and examples
insyt-secure --help
```

The help command provides detailed information about all parameters, their defaults, and usage examples directly in your terminal.

### Advanced Options

```bash
# Run with all options (for a single project)
insyt-secure \
  --projects "your-project-id-123:your-api-key-xyz" \
  --max-workers 10 \
  --execution-timeout 60 \
  --allowed-ips "192.168.1.1,10.0.0.1:3456"
```

### Logging Options

By default, logs are user-friendly and redact sensitive information. You can customize logging behavior:

```bash
# Enable more detailed debug logs
insyt-secure --projects "your-project-id:your-api-key" --debug

# Show more verbose logs including from third-party libraries
insyt-secure --projects "your-project-id:your-api-key" --verbose

# Output logs in JSON format (useful for log processing systems)
insyt-secure --projects "your-project-id:your-api-key" --json-logs

# Disable redaction of sensitive information (not recommended for production)
insyt-secure --projects "your-project-id:your-api-key" --show-sensitive
```

You can also control the log level via environment variables:

```bash
# Set log level using environment variable
INSYT_LOG_LEVEL=DEBUG insyt-secure --projects "your-project-id:your-api-key"

# Configure projects using environment variable (optional alternative to --projects)
INSYT_PROJECTS="project-id-1:api-key-1,project-id-2:api-key-2" insyt-secure
```

## Cross-Platform Compatibility

Insyt Secure is designed to work seamlessly on all major platforms:

- **Windows**: Fully supported natively, no additional configuration needed
- **macOS**: Fully supported
- **Linux**: Fully supported

The service uses paho-mqtt with a platform-agnostic implementation to ensure consistent behavior across all operating systems.

## Command Line Parameters

| Parameter | Description | Default |
|-----------|-------------|---------|
| `--projects` | Comma-separated list of project-id:api-key pairs for managing one or more projects | - |
| `--max-workers` | Maximum number of concurrent code executions per project | 5 |
| `--execution-timeout` | Default maximum execution time in seconds per code snippet | 30 |
| `--allowed-ips` | Comma-separated list of allowed IPs/hostnames | All allowed |
| `--verbose` | Enable more verbose logs | False |
| `--debug` | Enable debug level logging | False |
| `--json-logs` | Output logs in JSON format | False |
| `--show-sensitive` | Show sensitive information in logs | False |

## Credentials Management

This service automatically retrieves and manages connection credentials:

1. When the service starts, it gets credentials from the Insyt API for each configured project
2. If the connection drops or credentials expire, it automatically requests new credentials
3. Each project's credentials are managed independently

## Project Management

Insyt Secure supports managing one or more projects simultaneously:

### Configuration Options

You can configure projects in two ways:

1. **Command line parameter**:
   ```bash
   insyt-secure --projects "project-id-1:api-key-1,project-id-2:api-key-2"
   ```

2. **Environment variable**:
   ```bash
   export INSYT_PROJECTS="project-id-1:api-key-1,project-id-2:api-key-2"
   insyt-secure
   ```

### Benefits of Multi-Project Mode

- **Resource Efficiency**: All projects share a single process but maintain independent connections
- **Simplified Management**: Manage multiple projects from a single service instance
- **Independent Reconnection**: Each project can reconnect independently if its connection fails
- **Shared DNS Cache**: All projects benefit from the DNS cache for improved resilience

### API Usage for Multiple Projects

You can also use the project management programmatically:

```python
from insyt_secure.project_manager import ProjectManager
import asyncio

async def run_projects():
    manager = ProjectManager()
    
    # Add one or more projects
    await manager.add_project("project-id-1", "api-key-1", "https://api.account.insyt.co/")
    await manager.add_project("project-id-2", "api-key-2", "https://api.account.insyt.co/")
    
    # Start the manager
    await manager.start()
    
    # Keep the service running
    while True:
        await asyncio.sleep(60)

if __name__ == "__main__":
    asyncio.run(run_projects())
```

## Use Cases

### Data Processing Microservice

Perfect for running data transformation code that connects to various data sources:

```bash
insyt-secure --projects "your-project-id:your-api-key" --max-workers 15
```

### Secure Environment for Code Testing

Create a sandboxed environment with restricted network access:

```bash
insyt-secure --projects "your-project-id:your-api-key" \
  --allowed-ips "10.0.0.1,192.168.1.100" --execution-timeout 20
```

### Multi-Project Service

Run a single service that executes code for multiple projects:

```bash
insyt-secure --projects "PRJ-A235466:api-key-1,PRJ-A235477:api-key-2,ai-models:api-key-3" \
  --max-workers 10 --execution-timeout 60
```

### Containerized Deployment

```bash
docker run -d --name insyt-secure \
  insyt-secure insyt-secure \
  --projects "your-project-id:your-api-key"
```

## System Requirements and Dependencies

Insyt Secure is designed with a modular dependency structure to minimize installation size and resource usage. Below is a breakdown of what's included in each installation option:

### Core Dependencies (included in base install)

The base installation includes:
- HTTP client capabilities via `httpx`
- MQTT connectivity via `paho-mqtt`
- Basic data science libraries: NumPy, Pandas, SciPy
- JSON and date handling utilities

### Optional Dependencies

#### SQL Database Connectors
- `postgres`: High-performance PostgreSQL client (asyncpg)
- `mysql`: MySQL client libraries
- `mssql`: Microsoft SQL Server connectivity
- `oracle`: Oracle database connectivity
- `clickhouse`: ClickHouse analytics database client
- `snowflake`: Snowflake data warehouse client
- `duckdb`: Embedded analytical database

#### NoSQL Database Connectors
- `mongodb`: MongoDB client with async support
- `redis`: Redis client
- `cassandra`: Apache Cassandra and ScyllaDB clients
- `neo4j`: Neo4j graph database client
- `elasticsearch`: Elasticsearch search engine client
- `couchdb`: CouchDB document database client

#### Vector Database Connectors
- `pinecone`: Pinecone vector database client
- `qdrant`: Qdrant vector search engine client
- `milvus`: Milvus vector database client
- `weaviate`: Weaviate vector search engine
- `chroma`: ChromaDB for AI embeddings
- `faiss`: Facebook AI Similarity Search

#### Cloud Services
- `aws`: AWS SDK (boto3) with S3, Dynamo, etc.
- `azure`: Azure clients for Cosmos DB, Blob Storage, etc.

#### Messaging Systems
- `kafka`: Apache Kafka client
- `pulsar`: Apache Pulsar client
- `rabbitmq`: RabbitMQ client

### Performance Considerations

The base installation already includes the core ML libraries (numpy, pandas, etc.). If you're installing on a resource-constrained environment, consider using only the specific connector extensions you need rather than the broader categories.

For production deployments, we recommend specifying exact dependencies rather than using broader categories:

```bash
# Good (minimal dependencies)
pip install "insyt-secure[postgres,redis]"

# Less efficient (pulls in many unused dependencies)
pip install "insyt-secure[rdbms,nosql]"
```

### Platform-Specific Installation

Insyt Secure is designed to work on all major platforms without modification:

#### Windows

```bash
# Install on Windows
pip install insyt-secure

# Run (in PowerShell or Command Prompt)
insyt-secure --projects "your-project-id-123:your-api-key-xyz"
```

#### macOS/Linux

```bash
# Install on macOS/Linux
pip install insyt-secure

# Run
insyt-secure --projects "your-project-id-123:your-api-key-xyz"
```

#### Docker

```bash
# Create a simple Dockerfile
echo 'FROM python:3.10-slim
RUN pip install insyt-secure
ENTRYPOINT ["insyt-secure"]' > Dockerfile

# Build the Docker image
docker build -t insyt-secure .

# Run in Docker
docker run insyt-secure --projects "your-project-id-123:your-api-key-xyz"
```

#### Platform-Specific Considerations

- **Windows**: Works natively without WSL, using a cross-platform MQTT implementation
- **MacOS/Linux**: All features fully supported
- **Docker**: Ideal for deployment in containerized environments


#### Installing on a Fresh Ubuntu Instance

Follow these steps to set up insyt-secure on a fresh Ubuntu installation:

```bash
# Step 1: Install required packages
sudo apt update
sudo apt install python3-pip
sudo apt install python3-venv python3-full

# Step 2: Create a directory for your project (optional)
mkdir myproject
cd myproject

# Step 3: Create the virtual environment
python3 -m venv venv

# Step 4: Activate the virtual environment
source venv/bin/activate
# Your command prompt should change to show the virtual environment name
# Example: (venv) ubuntu@ip-100-1-11-111:~/myproject$

# Step 5: Install insyt-secure with desired extensions
pip install insyt-secure[mysql,postgres]
pip install python-dotenv
```

**Notes:**
1. To run the service, use the following command:
```bash
insyt-secure --projects "your-project-id-123:your-api-key-xyz"
```
or run the service in the background:
```bash
nohup insyt-secure --projects "your-project-id-123:your-api-key-xyz" &
```

2. Remember to activate the virtual environment each time you want to use this package in a new terminal session.
```bash
source venv/bin/activate
```

3. To deactivate the virtual environment when you're done:
```bash
deactivate
```

4. To stop the service, use the following command:
```bash
pkill -f insyt-secure
```

5. To check the status of the service, use the following command:
```bash
ps aux | grep insyt-secure
```



