ParallelLLC/algorithmic_trading

1name: Algorithmic Trading CI/CD Pipeline2 3on:4  push:5    branches: [ main, dev ]6  pull_request:7    branches: [ main ]8  release:9    types: [ published ]10 11env:12  DOCKER_IMAGE: dataen10/algorithmic_trading13  PYTHON_VERSION: '3.11'14 15jobs:16  # Quality Assurance17  quality-check:18    name: Code Quality & Security19    runs-on: ubuntu-latest20    21    steps:22    - name: Checkout code23      uses: actions/checkout@v424      25    - name: Set up Python26      uses: actions/setup-python@v527      with:28        python-version: ${{ env.PYTHON_VERSION }}29        30    - name: Install dependencies31      run: |32        python -m pip install --upgrade pip33        pip install -r requirements.txt34        pip install flake8 black isort bandit safety35        36    - name: Code formatting check37      run: |38        black --check --diff .39        isort --check-only --diff .40        41    - name: Linting42      run: |43        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics44        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=88 --statistics45        46    - name: Security scan47      run: |48        bandit -r . -f json -o bandit-report.json || true49        safety check --json --output safety-report.json || true50        51    - name: Upload security reports52      uses: actions/upload-artifact@v453      with:54        name: security-reports55        path: |56          bandit-report.json57          safety-report.json58 59  # Testing60  test:61    name: Run Test Suite62    runs-on: ubuntu-latest63    needs: quality-check64    65    strategy:66      matrix:67        python-version: ['3.9', '3.10', '3.11']68        69    steps:70    - name: Checkout code71      uses: actions/checkout@v472      73    - name: Set up Python ${{ matrix.python-version }}74      uses: actions/setup-python@v575      with:76        python-version: ${{ matrix.python-version }}77        78    - name: Install dependencies79      run: |80        python -m pip install --upgrade pip81        pip install -r requirements.txt82        83    - name: Run tests with coverage84      run: |85        pytest tests/ -v --cov=agentic_ai_system --cov-report=xml --cov-report=html86        87    - name: Upload coverage reports88      uses: codecov/codecov-action@v589      with:90        file: ./coverage.xml91        flags: unittests92        name: codecov-umbrella93        94    - name: Upload test artifacts95      uses: actions/upload-artifact@v496      with:97        name: test-results-${{ matrix.python-version }}98        path: |99          htmlcov/100          .pytest_cache/101 102  # FinRL Model Training & Validation103  model-training:104    name: FinRL Model Training105    runs-on: ubuntu-latest106    needs: test107    if: github.ref == 'refs/heads/main'108    109    steps:110    - name: Checkout code111      uses: actions/checkout@v4112      113    - name: Set up Python114      uses: actions/setup-python@v5115      with:116        python-version: ${{ env.PYTHON_VERSION }}117        118    - name: Install dependencies119      run: |120        python -m pip install --upgrade pip121        pip install -r requirements.txt122        123    - name: Train FinRL model124      run: |125        python -c "126        from agentic_ai_system.finrl_agent import FinRLAgent, FinRLConfig127        from agentic_ai_system.data_ingestion import load_data, load_config128        129        config = load_config()130        data = load_data(config)131        132        agent = FinRLAgent(FinRLConfig(algorithm='PPO', learning_rate=0.0003))133        result = agent.train(data=data, config=config, total_timesteps=10000)134        print(f'Training completed: {result}')135        "136        137    - name: Upload trained model138      uses: actions/upload-artifact@v4139      with:140        name: finrl-model141        path: models/finrl_best/142 143  # Docker Build & Test144  docker-build:145    name: Docker Build & Test146    runs-on: ubuntu-latest147    needs: [test, model-training]148    149    steps:150    - name: Checkout code151      uses: actions/checkout@v4152      153    - name: Set up Docker Buildx154      uses: docker/setup-buildx-action@v3155      156    - name: Build Docker image157      run: |158        docker build -t ${{ env.DOCKER_IMAGE }}:test .159        160    - name: Test Docker image161      run: |162        docker run --rm ${{ env.DOCKER_IMAGE }}:test python -c "163        from agentic_ai_system.main import main164        print('Docker image test passed')165        "166        167    - name: Save Docker image168      run: |169        docker save ${{ env.DOCKER_IMAGE }}:test -o /tmp/docker-image.tar170        171    - name: Upload Docker image172      uses: actions/upload-artifact@v4173      with:174        name: docker-image175        path: /tmp/docker-image.tar176 177  # Docker Hub Push178  docker-push:179    name: Push to Docker Hub180    runs-on: ubuntu-latest181    needs: docker-build182    if: github.ref == 'refs/heads/main'183    184    steps:185    - name: Checkout code186      uses: actions/checkout@v4187      188    - name: Set up Docker Buildx189      uses: docker/setup-buildx-action@v3190      191    - name: Login to Docker Hub192      uses: docker/login-action@v3193      with:194        username: ${{ secrets.DOCKERHUB_USERNAME }}195        password: ${{ secrets.DOCKERHUB_TOKEN }}196        197    - name: Extract metadata198      id: meta199      uses: docker/metadata-action@v5200      with:201        images: ${{ env.DOCKER_IMAGE }}202        tags: |203          type=ref,event=branch204          type=ref,event=pr205          type=semver,pattern={{version}}206          type=semver,pattern={{major}}.{{minor}}207          type=sha208          209    - name: Build and push Docker image210      uses: docker/build-push-action@v6211      with:212        context: .213        push: true214        tags: ${{ steps.meta.outputs.tags }}215        labels: ${{ steps.meta.outputs.labels }}216        cache-from: type=gha217        cache-to: type=gha,mode=max218 219  # Documentation Generation220  docs:221    name: Generate Documentation222    runs-on: ubuntu-latest223    needs: test224    if: github.ref == 'refs/heads/main'225    226    steps:227    - name: Checkout code228      uses: actions/checkout@v4229      230    - name: Set up Python231      uses: actions/setup-python@v5232      with:233        python-version: ${{ env.PYTHON_VERSION }}234        235    - name: Install dependencies236      run: |237        python -m pip install --upgrade pip238        pip install -r requirements.txt239        pip install sphinx sphinx-rtd-theme240        241    - name: Generate API documentation242      run: |243        sphinx-apidoc -o docs/source agentic_ai_system/244        sphinx-build -b html docs/source docs/build/html245        246    - name: Deploy to GitHub Pages247      uses: peaceiris/actions-gh-pages@v4248      if: github.ref == 'refs/heads/main'249      with:250        github_token: ${{ secrets.GITHUB_TOKEN }}251        publish_dir: ./docs/build/html252 253  # Performance Testing254  performance:255    name: Performance & Load Testing256    runs-on: ubuntu-latest257    needs: docker-build258    if: github.ref == 'refs/heads/main'259    260    steps:261    - name: Checkout code262      uses: actions/checkout@v4263      264    - name: Set up Python265      uses: actions/setup-python@v5266      with:267        python-version: ${{ env.PYTHON_VERSION }}268        269    - name: Install dependencies270      run: |271        python -m pip install --upgrade pip272        pip install -r requirements.txt273        pip install locust274        275    - name: Run performance tests276      run: |277        python -c "278        from agentic_ai_system.data_ingestion import load_data, load_config279        from agentic_ai_system.strategy_agent import StrategyAgent280        import time281        282        config = load_config()283        data = load_data(config)284        285        agent = StrategyAgent()286        287        start_time = time.time()288        for _ in range(100):289            signals = agent.generate_signals(data)290        end_time = time.time()291        292        avg_time = (end_time - start_time) / 100293        print(f'Average signal generation time: {avg_time:.4f} seconds')294        assert avg_time < 0.1, 'Performance threshold exceeded'295        "296        297    - name: Upload performance report298      uses: actions/upload-artifact@v4299      with:300        name: performance-report301        path: performance-results.json302 303  # Security & Compliance304  security:305    name: Security & Compliance Check306    runs-on: ubuntu-latest307    needs: test308    309    steps:310    - name: Checkout code311      uses: actions/checkout@v4312      313    - name: Run Trivy vulnerability scanner314      uses: aquasecurity/trivy-action@master315      with:316        image-ref: ${{ env.DOCKER_IMAGE }}:test317        format: 'sarif'318        output: 'trivy-results.sarif'319        320    - name: Upload Trivy scan results321      uses: github/codeql-action/upload-sarif@v3322      if: always()323      with:324        sarif_file: 'trivy-results.sarif'325        326    - name: Check for secrets in code327      run: |328        pip install detect-secrets329        detect-secrets scan --baseline .secrets.baseline330        331    - name: Trading compliance check332      run: |333        python -c "334        from agentic_ai_system.execution_agent import ExecutionAgent335        from agentic_ai_system.config import load_config336        337        config = load_config()338        agent = ExecutionAgent(config)339        340        # Check risk management settings341        assert config['risk']['max_position'] <= 100, 'Position limit too high'342        assert config['risk']['max_drawdown'] <= 0.05, 'Drawdown limit too high'343        print('Compliance checks passed')344        "345 346  # Notification347  notify:348    name: Notify Team349    runs-on: ubuntu-latest350    needs: [docker-push, docs, performance, security]351    if: always()352    353    steps:354    - name: Notify on success355      if: success()356      run: |357        echo "✅ CI/CD Pipeline completed successfully!"358        echo "🚀 New version deployed to Docker Hub"359        echo "📚 Documentation updated"360        echo "🔒 Security checks passed"361        362    - name: Notify on failure363      if: failure()364      run: |365        echo "❌ CI/CD Pipeline failed!"366        echo "Please check the logs for details"367        368    - name: Send Slack notification369      if: always()370      uses: 8398a7/action-slack@v3371      with:372        status: ${{ job.status }}373        channel: '#trading-alerts'374      env:375        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}