16.1. CI/CD Tools

16.1.1. Static Analysis

Table 16.1. Static Analysis

Tool

Description

pylama

pylint

pyflakes

flake8

SonarQube

SonarScanner

SonarLint

16.1.2. Security

Table 16.2. Security

Tool

Description

safety

bandit

16.1.3. Distributing and Packaging

Table 16.3. Distributing and Packaging

Tool

Description

pipenv

Frozen env

venv

16.1.4. Code Style and Practices

Table 16.4. Code Style and Practices

Tool

Description

pycodestyle

pydocestyle

eradicate

Remove commented code

isort

cloc

Count Lines of Code

16.1.5. Code complexity and Coverage

Table 16.5. Code complexity and Coverage

Tool

Description

mccabe

radon

coverage

16.1.6. Testing

Table 16.6. Testing

Tool

Description

doctest

unittest

selenium

behave

mutpy

tox

pytest

16.1.7. Type Checking

Table 16.7. Type Checking

Tool

Description

mypy

pyre-check

pytype

monkeytype

pyannotate

16.1.8. Database Schema Migration

Table 16.8. Database Schema Migration

Tool

Description

SQLAlchemy

django.migrations

Liquibase

FlywayDB

16.1.9. Running

import os
import subprocess
import logging
from config import APPS, STDOUT_DIRECTORY, PROJECT_DIRECTORY


logging.basicConfig(
    level=logging.DEBUG,
    format='[%(asctime).19s] %(levelname)s\t %(message)s')

# pip install pylama
# pip install radon
# pip install bandit
# pip install pycodestyle
# pip install eradicate
# pip install mccabe
# pip install pyflakes
# pip install pylint
# pip install isort
# pip install pydocstyle
#
# ## setup.cfg
#
# [pylama:pycodestyle]
# max_line_length = 300


COMMANDS = [
    {'name': 'bandit',      'timeout': 180, 'command': 'bandit --recursive {directory}'},
    {'name': 'cloc',        'timeout': 180, 'command': 'cloc --fullpath --not-match-d="(migrations|tinymce|jquery)" {directory}'},
    {'name': 'pycodestyle', 'timeout': 180, 'command': 'pylama --format parsable --linters pycodestyle --skip="*/migrations/*" {directory}'},
    {'name': 'eradicate',   'timeout': 180, 'command': 'pylama --format parsable --linters eradicate --skip="*/migrations/*" {directory}'},
    {'name': 'mccabe',      'timeout': 180, 'command': 'pylama --format parsable --linters mccabe --skip="*/migrations/*" {directory}'},
    {'name': 'radon',       'timeout': 180, 'command': 'pylama --format parsable --linters radon --skip="*/migrations/*" {directory}'},
    {'name': 'pyflakes',    'timeout': 180, 'command': 'pylama --format parsable --linters pyflakes --skip="*/migrations/*" {directory}'},
    {'name': 'isort',       'timeout': 180, 'command': 'pylama --format parsable --linters isort --skip="*/migrations/*" {directory}'},
    {'name': 'pydocstyle',  'timeout': 180, 'command': 'pylama --format parsable --linters pydocstyle --skip="*/migrations/*" --ignore=D100,D101,D102,D103,D104,D105,D106,D107,D200,D205,D212,D400,D404 {directory}'},
    {'name': 'pylint',      'timeout': 180, 'command': 'pylama --format parsable --linters pylint --skip="*/migrations/*" {directory}'},
]


os.chdir(PROJECT_DIRECTORY)


for app_name in APPS:
    logging.warning('Processing: "{}"'.format(app_name))
    stdout_dir = os.path.join(STDOUT_DIRECTORY, app_name)
    os.makedirs(stdout_dir, exist_ok=True)

    for command in COMMANDS:
        linter = command['name']
        cmd = command['command'].format(directory=app_name)
        header = '``{}``'.format(linter)
        underscore = '-' * len(header)
        stdout_file = os.path.join(stdout_dir, linter+'.txt')
        logging.info(cmd)

        try:
            result = subprocess.run(
                cmd,
                shell=True,
                timeout=command['timeout'],
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                encoding='utf-8')
        except subprocess.TimeoutExpired:
            logging.error('Timeout exceeded')
            continue

        if result.stderr:
            logging.debug(result.stderr)

        with open(stdout_file, mode='w') as file:
            file.write(result.stdout)


HEADER = """

Static Analysis
===============
"""

REPORT = """
.. code-block:: console
    :caption: Running static analysis ``{linter}`` for module ``{app}``

    {command}

.. literalinclude:: /_stdout/{app}/{linter}.txt
    :caption: Result of static analysis ``{linter}`` for module ``{app}``
    :language: text
"""

for app_name in APPS:
    logging.warning('Adding reports: "{}"'.format(app_name))
    report_file = os.path.join(STDOUT_DIRECTORY, '..', 'code-review', app_name + '.rst')

    with open(report_file, mode='a') as file:
        file.write(HEADER)
        file.write('\n')

    for command in COMMANDS:
        linter = command['name']
        cmd = command['command'].format(directory=app_name)
        header = '``{}``'.format(linter)
        underscore = '-' * len(header)

        with open(report_file, mode='a') as file:
            file.write(header)
            file.write('\n')
            file.write(underscore)
            file.write('\n')
            file.write(REPORT.format(linter=linter, command=cmd, app=app_name))
            file.write('\n')